@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;
|
|
1
|
+
{"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAYvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,YACE,QAAsE,EACtE,aAA2E;QAE3E,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,qBAAqB;QACvB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,aAAa,CAAC,CAAC;IACjE,CAAC;IAED,aAAa,CAAC,mBAA2B,EAAE,aAA6B;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,cAAc,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,mBAA2B;QAC3D,OAAO,IAAI,CAAC,uBAAuB,CAAwB,kBAAkB,mBAAmB,GAAG,CAAC,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,mBAA2B,EAC3B,aAA6B,EAC7B,OAA6B;QAE7B,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,WAAW,CACtE;YACE,cAAc,EAAE,CAAC;SAClB,EACD;YACE,uBAAuB,EAAE,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC9D,IAAI,EAAE,gBAAgB;YACtB,OAAO;SACR,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,mBAA2B;QAC5C,OAAO,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAuD,CAAC;IACtG,CAAC;IAED,WAAW,CAAC,mBAA2B;QACrC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,cAAc,CAAC,mBAA2B,EAAE,OAAyB;QACnE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,YAAY,CAAC,mBAA2B;QACtC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,UAAU,CAAC,mBAA2B,EAAE,YAAgC;QACtE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,eAAe,mBAAmB,IAAI,YAAY,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,2BAA2B,CAAC,mBAA2B;QACrD,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAA8B,kCAAkC,mBAAmB,EAAE,CAAC,CAAC;IAClH,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,IAAI,CAAC,uBAAuB,CAAC,eAAe,mBAAmB,GAAG,CAAC,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,mBAA2B;QAC7D,MAAM,MAAM,GAAG,mBAAmB,mBAAmB,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAE/D,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACtC,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SACxD,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,mBAA2B;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;SACf,EACD;YACE,IAAI,EAAE,eAAe;SACtB,CACF,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,qBAAqB,EAAE,CAAC;SACzB,EACD;YACE,uBAAuB,EAAE,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACrE,IAAI,EAAE,uBAAuB;SAC9B,CACF,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
2
|
+
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
3
|
+
import { BucketDataDocumentV3 } from './models.js';
|
|
4
|
+
/**
|
|
5
|
+
* Serialize a non-empty list of operations in strictly ascending `o` order.
|
|
6
|
+
* Preserving that order establishes the {@link BucketDataDocumentV3} range
|
|
7
|
+
* invariants used by metadata-only queries.
|
|
8
|
+
*/
|
|
9
|
+
export declare function serializeBucketData(bucket: string, operations: BucketDataDoc[], options?: {
|
|
10
|
+
targetOp?: InternalOpId | null;
|
|
11
|
+
}): BucketDataDocumentV3;
|
|
12
|
+
export declare function loadBucketDataDocument(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): Generator<BucketDataDoc>;
|
|
13
|
+
export declare function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { bson } from '@powersync/service-core';
|
|
3
|
+
/**
|
|
4
|
+
* Serialize a non-empty list of operations in strictly ascending `o` order.
|
|
5
|
+
* Preserving that order establishes the {@link BucketDataDocumentV3} range
|
|
6
|
+
* invariants used by metadata-only queries.
|
|
7
|
+
*/
|
|
8
|
+
export function serializeBucketData(bucket, operations, options) {
|
|
9
|
+
const minOp = operations[0].o;
|
|
10
|
+
const maxOp = operations[operations.length - 1].o;
|
|
11
|
+
let totalChecksum = 0n;
|
|
12
|
+
let maxTargetOp = options?.targetOp ?? null;
|
|
13
|
+
let hasClearOp = false;
|
|
14
|
+
const ops = operations.map((op) => {
|
|
15
|
+
totalChecksum += op.checksum;
|
|
16
|
+
if (op.op == 'CLEAR') {
|
|
17
|
+
hasClearOp = true;
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
o: op.o,
|
|
21
|
+
op: op.op,
|
|
22
|
+
source_table: op.source_table,
|
|
23
|
+
source_key: op.source_key,
|
|
24
|
+
table: op.table,
|
|
25
|
+
row_id: op.row_id,
|
|
26
|
+
checksum: op.checksum,
|
|
27
|
+
data: op.data
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
const size = bson.calculateObjectSize(ops);
|
|
31
|
+
return {
|
|
32
|
+
_id: {
|
|
33
|
+
b: bucket,
|
|
34
|
+
o: maxOp
|
|
35
|
+
},
|
|
36
|
+
min_op: minOp,
|
|
37
|
+
checksum: totalChecksum,
|
|
38
|
+
count: operations.length,
|
|
39
|
+
size,
|
|
40
|
+
target_op: maxTargetOp,
|
|
41
|
+
...(hasClearOp ? { has_clear_op: true } : {}),
|
|
42
|
+
ops
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function* loadBucketDataDocument(context, doc) {
|
|
46
|
+
const { _id, ops } = doc;
|
|
47
|
+
if (!ops) {
|
|
48
|
+
throw new ServiceAssertionError(`Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`);
|
|
49
|
+
}
|
|
50
|
+
const bucketKey = {
|
|
51
|
+
...context,
|
|
52
|
+
bucket: _id.b
|
|
53
|
+
};
|
|
54
|
+
for (const op of ops) {
|
|
55
|
+
yield {
|
|
56
|
+
bucketKey,
|
|
57
|
+
o: op.o,
|
|
58
|
+
op: op.op,
|
|
59
|
+
source_table: op.source_table,
|
|
60
|
+
source_key: op.source_key,
|
|
61
|
+
table: op.table,
|
|
62
|
+
row_id: op.row_id,
|
|
63
|
+
checksum: op.checksum,
|
|
64
|
+
data: op.data
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export function maxOpId(a, b) {
|
|
69
|
+
if (a == null) {
|
|
70
|
+
return b ?? null;
|
|
71
|
+
}
|
|
72
|
+
if (b == null) {
|
|
73
|
+
return a ?? null;
|
|
74
|
+
}
|
|
75
|
+
return a > b ? a : b;
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=bucket-format.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bucket-format.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/bucket-format.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAgB,MAAM,yBAAyB,CAAC;AAI7D;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,UAA2B,EAC3B,OAA4C;IAE5C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAElD,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,WAAW,GAAwB,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC;IACjE,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,MAAM,GAAG,GAAsB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACnD,aAAa,IAAI,EAAE,CAAC,QAAQ,CAAC;QAE7B,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YACrB,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,OAAO;YACL,CAAC,EAAE,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,IAAI,EAAE,EAAE,CAAC,IAAI;SACd,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,KAAK;SACT;QACD,MAAM,EAAE,KAAK;QACb,QAAQ,EAAE,aAAa;QACvB,KAAK,EAAE,UAAU,CAAC,MAAM;QACxB,IAAI;QACJ,SAAS,EAAE,WAAW;QACtB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,sBAAsB,CACrC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,qBAAqB,CAC7B,sDAAsD,GAAG,CAAC,CAAC,oEAAoE,CAChI,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG;QAChB,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,CAAC,CAAC;KACd,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM;YACJ,SAAS;YACT,CAAC,EAAE,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,IAAI,EAAE,EAAE,CAAC,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,CAAkC,EAAE,CAAkC;IAC5F,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,IAAI,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,IAAI,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
2
|
+
export declare const DEFAULT_MAX_DOC_SIZE_BYTES: number;
|
|
3
|
+
/**
|
|
4
|
+
* Split an array of bucket data operations into chunks where each chunk's
|
|
5
|
+
* total data size stays close to `maxDocSizeBytes`.
|
|
6
|
+
*
|
|
7
|
+
* The threshold is a target, not a hard ceiling — a single operation whose
|
|
8
|
+
* `data` field exceeds the limit is placed in its own chunk and is NOT split.
|
|
9
|
+
*/
|
|
10
|
+
export declare function chunkBucketData(operations: BucketDataDoc[], maxDocSizeBytes?: number): BucketDataDoc[][];
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { bson } from '@powersync/service-core';
|
|
2
|
+
export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
|
|
3
|
+
/**
|
|
4
|
+
* Split an array of bucket data operations into chunks where each chunk's
|
|
5
|
+
* total data size stays close to `maxDocSizeBytes`.
|
|
6
|
+
*
|
|
7
|
+
* The threshold is a target, not a hard ceiling — a single operation whose
|
|
8
|
+
* `data` field exceeds the limit is placed in its own chunk and is NOT split.
|
|
9
|
+
*/
|
|
10
|
+
export function chunkBucketData(operations, maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES) {
|
|
11
|
+
const chunks = [];
|
|
12
|
+
let currentChunk = [];
|
|
13
|
+
let currentSize = 0;
|
|
14
|
+
for (const op of operations) {
|
|
15
|
+
// Add 5 to account for array index up to 4 digits (9999) plus null byte.
|
|
16
|
+
const opSize = bson.calculateObjectSize(op) + 5;
|
|
17
|
+
if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
|
|
18
|
+
chunks.push(currentChunk);
|
|
19
|
+
currentChunk = [];
|
|
20
|
+
currentSize = 0;
|
|
21
|
+
}
|
|
22
|
+
currentChunk.push(op);
|
|
23
|
+
currentSize += opSize;
|
|
24
|
+
}
|
|
25
|
+
if (currentChunk.length > 0) {
|
|
26
|
+
chunks.push(currentChunk);
|
|
27
|
+
}
|
|
28
|
+
return chunks;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=chunking.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAG/C,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAE7D;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,UAA2B,EAC3B,eAAe,GAAG,0BAA0B;IAE5C,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,YAAY,GAAoB,EAAE,CAAC;IACvC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,yEAAyE;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAEhD,IAAI,WAAW,GAAG,MAAM,GAAG,eAAe,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1B,YAAY,GAAG,EAAE,CAAC;YAClB,WAAW,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,WAAW,IAAI,MAAM,CAAC;IACxB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import { InternalOpId, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
|
|
2
|
-
import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
|
|
1
|
+
import { InternalOpId, PersistedDefinitionMapping, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
|
|
2
|
+
import { BucketDefinitionId, ParameterIndexId, ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
|
-
import {
|
|
5
|
-
import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
|
|
4
|
+
import { BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, OpType, ReplicaId, SourceTableDocumentSnapshotStatus, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
|
|
6
5
|
/**
|
|
7
6
|
* Embedded in sync_rules.sync_configs.
|
|
8
7
|
*/
|
|
9
|
-
export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields
|
|
8
|
+
export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields {
|
|
10
9
|
_id: bson.ObjectId;
|
|
11
10
|
/**
|
|
12
11
|
* If false, we cannot create any checkpoints.
|
|
@@ -30,6 +29,29 @@ export interface ReplicationStreamDocumentV3 extends SyncRuleDocumentBase {
|
|
|
30
29
|
* but the model allows multiple configs in any state.
|
|
31
30
|
*/
|
|
32
31
|
sync_configs: SyncRuleConfigStateV3[];
|
|
32
|
+
/**
|
|
33
|
+
* The monotonic head of the stream's op sequence: the highest op id persisted to bucket data,
|
|
34
|
+
* whether or not yet covered by a checkpoint.
|
|
35
|
+
*
|
|
36
|
+
* This is shared across all sync configs of the stream (they share the global op sequence).
|
|
37
|
+
* It is never cleared, only `$max`-advanced. A newly-appended config that replicates nothing
|
|
38
|
+
* adopts this value as its checkpoint rather than starting at 0.
|
|
39
|
+
*
|
|
40
|
+
* Stored as a mongo Long, nullable.
|
|
41
|
+
*/
|
|
42
|
+
last_persisted_op?: bigint | null;
|
|
43
|
+
/**
|
|
44
|
+
* The stream's replication position: all source changes up to this LSN have been processed,
|
|
45
|
+
* and the resulting ops persisted. Replication resumes from here.
|
|
46
|
+
*
|
|
47
|
+
* Like {@link last_persisted_op}, this is shared across all sync configs of the stream -
|
|
48
|
+
* per-config last_checkpoint_lsn values are consistency markers, not replication positions.
|
|
49
|
+
*
|
|
50
|
+
* Set via setResumeLsn() (snapshot start, and per-batch progress during streaming), and
|
|
51
|
+
* advanced on every commit/keepalive - including checkpoint-blocked ones, since commit
|
|
52
|
+
* flushes first and blocking only delays consistency markers, not data persistence.
|
|
53
|
+
*/
|
|
54
|
+
resume_lsn?: string | null;
|
|
33
55
|
}
|
|
34
56
|
/**
|
|
35
57
|
* Static sync config definition.
|
|
@@ -50,16 +72,7 @@ export interface SyncConfigDefinition {
|
|
|
50
72
|
replication_stream_id: number;
|
|
51
73
|
content: string;
|
|
52
74
|
serialized_plan?: SerializedSyncPlan | null;
|
|
53
|
-
rule_mapping:
|
|
54
|
-
/**
|
|
55
|
-
* Map of uniqueName -> id, unique per replication stream.
|
|
56
|
-
*/
|
|
57
|
-
definitions: Record<string, string>;
|
|
58
|
-
/**
|
|
59
|
-
* Map of (lookupName, queryId) -> id, unique per replication stream.
|
|
60
|
-
*/
|
|
61
|
-
parameter_indexes: Record<string, string>;
|
|
62
|
-
};
|
|
75
|
+
rule_mapping: PersistedDefinitionMapping;
|
|
63
76
|
}
|
|
64
77
|
export interface CurrentBucketV3 extends CurrentBucket {
|
|
65
78
|
def: BucketDefinitionId;
|
|
@@ -83,13 +96,21 @@ export interface CurrentDataDocumentV3 {
|
|
|
83
96
|
export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {
|
|
84
97
|
}
|
|
85
98
|
export type BucketDataKeyV3 = BucketDataKey;
|
|
86
|
-
export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
|
|
87
|
-
_id: BucketDataKeyV3;
|
|
88
|
-
}
|
|
89
|
-
export declare function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3;
|
|
90
|
-
export declare function loadBucketDataDocumentV3(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): BucketDataDoc;
|
|
91
99
|
export declare function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
|
|
92
|
-
export interface
|
|
100
|
+
export interface ReplicaIdColumn {
|
|
101
|
+
name: string;
|
|
102
|
+
type_oid?: number;
|
|
103
|
+
type?: string;
|
|
104
|
+
}
|
|
105
|
+
export interface SourceTableDocumentV3 {
|
|
106
|
+
_id: bson.ObjectId;
|
|
107
|
+
connection_id: number;
|
|
108
|
+
relation_id: number | string | undefined;
|
|
109
|
+
schema_name: string;
|
|
110
|
+
table_name: string;
|
|
111
|
+
replica_id_columns: ReplicaIdColumn[];
|
|
112
|
+
snapshot_done: boolean;
|
|
113
|
+
snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
|
|
93
114
|
bucket_data_source_ids: BucketDefinitionId[];
|
|
94
115
|
parameter_lookup_source_ids: ParameterIndexId[];
|
|
95
116
|
latest_pending_delete?: InternalOpId | undefined;
|
|
@@ -99,3 +120,62 @@ export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
|
|
|
99
120
|
d: BucketDefinitionId;
|
|
100
121
|
};
|
|
101
122
|
}
|
|
123
|
+
export interface BucketOperation {
|
|
124
|
+
o: bigint;
|
|
125
|
+
op: OpType;
|
|
126
|
+
source_table?: bson.ObjectId;
|
|
127
|
+
source_key?: ReplicaId;
|
|
128
|
+
table?: string;
|
|
129
|
+
row_id?: string;
|
|
130
|
+
checksum: bigint;
|
|
131
|
+
data: string | null;
|
|
132
|
+
}
|
|
133
|
+
export interface StorageRef {
|
|
134
|
+
path: string;
|
|
135
|
+
file_size: number;
|
|
136
|
+
}
|
|
137
|
+
/** An S3 object that may be deleted once its grace period has elapsed. */
|
|
138
|
+
export interface ObjectStorageDeletionMarker {
|
|
139
|
+
_id: bson.ObjectId;
|
|
140
|
+
path: string;
|
|
141
|
+
delete_after: Date;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* A non-empty chunk of a bucket's ordered operation stream.
|
|
145
|
+
*
|
|
146
|
+
* All writers maintain these invariants:
|
|
147
|
+
*
|
|
148
|
+
* - `ops` is non-empty and strictly ordered by `o`.
|
|
149
|
+
* - `min_op` is `ops[0].o` and `_id.o` is `ops[ops.length - 1].o`.
|
|
150
|
+
* - Documents for the same bucket have ordered, disjoint operation ranges,
|
|
151
|
+
* although gaps between ranges are allowed.
|
|
152
|
+
* - `checksum` is the sum of every operation checksum and `count` is
|
|
153
|
+
* `ops.length`.
|
|
154
|
+
*
|
|
155
|
+
* Metadata-only checksum and data queries rely on these invariants to identify
|
|
156
|
+
* boundary-straddling documents without reading `ops`.
|
|
157
|
+
*/
|
|
158
|
+
export interface BucketDataDocumentV3 {
|
|
159
|
+
_id: BucketDataKey;
|
|
160
|
+
min_op: bigint;
|
|
161
|
+
checksum: bigint;
|
|
162
|
+
count: number;
|
|
163
|
+
size: number;
|
|
164
|
+
/**
|
|
165
|
+
* The greatest operation boundary that influenced this document's contents.
|
|
166
|
+
* Bucket-data reads propagate it so serving an earlier checkpoint can be
|
|
167
|
+
* invalidated when compaction changed the operations being returned.
|
|
168
|
+
*/
|
|
169
|
+
target_op?: bigint | null;
|
|
170
|
+
ops?: BucketOperation[];
|
|
171
|
+
storage_ref?: StorageRef;
|
|
172
|
+
/**
|
|
173
|
+
* Present (and always true) when this document contains a CLEAR operation.
|
|
174
|
+
* In that case, this is the first document in the bucket: all preceding
|
|
175
|
+
* documents have been removed.
|
|
176
|
+
*/
|
|
177
|
+
has_clear_op?: true;
|
|
178
|
+
}
|
|
179
|
+
export declare function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary;
|
|
180
|
+
export declare function deserializeParameterLookup(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[];
|
|
181
|
+
export declare function taggedBucketParameterDocumentToTagged(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
|
|
@@ -1,33 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
o: o
|
|
7
|
-
},
|
|
8
|
-
// List fields directly, so that we don't accidentally persist any unknown fields
|
|
9
|
-
op: document.op,
|
|
10
|
-
source_table: document.source_table,
|
|
11
|
-
source_key: document.source_key,
|
|
12
|
-
table: document.table,
|
|
13
|
-
row_id: document.row_id,
|
|
14
|
-
checksum: document.checksum,
|
|
15
|
-
data: document.data,
|
|
16
|
-
target_op: document.target_op
|
|
17
|
-
};
|
|
1
|
+
import { deserializeParameterLookup as deserializeParameterLookupCore } from '@powersync/service-core';
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
export function taggedBucketParameterDocumentToV3(document) {
|
|
4
|
+
const { index: _index, ...rest } = document;
|
|
5
|
+
return rest;
|
|
18
6
|
}
|
|
19
|
-
export function
|
|
20
|
-
|
|
21
|
-
return {
|
|
22
|
-
bucketKey: {
|
|
23
|
-
...context,
|
|
24
|
-
bucket: _id.b
|
|
25
|
-
},
|
|
26
|
-
o: _id.o,
|
|
27
|
-
...rest
|
|
28
|
-
};
|
|
7
|
+
export function serializeParameterLookup(lookup) {
|
|
8
|
+
return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
|
|
29
9
|
}
|
|
30
|
-
export function
|
|
10
|
+
export function deserializeParameterLookup(lookup, indexId) {
|
|
11
|
+
return [indexId, '', ...deserializeParameterLookupCore(lookup)];
|
|
12
|
+
}
|
|
13
|
+
export function taggedBucketParameterDocumentToTagged(document) {
|
|
31
14
|
const { index: _index, ...rest } = document;
|
|
32
15
|
return rest;
|
|
33
16
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,0BAA0B,IAAI,8BAA8B,EAC7D,MAAM,yBAAyB,CAAC;AAOjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AA2H7B,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC;AAwFD,MAAM,UAAU,wBAAwB,CAAC,MAA6B;IACpE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAmB,EAAE,OAAyB;IACvF,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,qCAAqC,CACnD,QAAuC;IAEvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { BucketDataDocumentV3, BucketOperation } from '../models.js';
|
|
2
|
+
import { ObjectStorage } from './ObjectStorage.js';
|
|
3
|
+
export declare class BucketDataObjectStorage {
|
|
4
|
+
private readonly storage;
|
|
5
|
+
constructor(storage: ObjectStorage);
|
|
6
|
+
store(path: string, ops: BucketOperation[]): Promise<{
|
|
7
|
+
fileSize: number;
|
|
8
|
+
}>;
|
|
9
|
+
retrieve(path: string, options: {
|
|
10
|
+
signal?: AbortSignal;
|
|
11
|
+
}): Promise<BucketOperation[]>;
|
|
12
|
+
delete(paths: string[]): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Load offloaded operations and patch them onto their MongoDB metadata documents.
|
|
16
|
+
* The object-storage implementation is responsible for limiting request
|
|
17
|
+
* concurrency across all callers.
|
|
18
|
+
*/
|
|
19
|
+
export declare function hydrateBucketDataDocuments(documents: BucketDataDocumentV3[], objectStorage: ObjectStorage | undefined, options: {
|
|
20
|
+
signal?: AbortSignal;
|
|
21
|
+
}): Promise<void>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
export class BucketDataObjectStorage {
|
|
4
|
+
storage;
|
|
5
|
+
constructor(storage) {
|
|
6
|
+
this.storage = storage;
|
|
7
|
+
}
|
|
8
|
+
async store(path, ops) {
|
|
9
|
+
const bsonBuffer = bson.serialize({ ops });
|
|
10
|
+
await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
|
|
11
|
+
return { fileSize: bsonBuffer.byteLength };
|
|
12
|
+
}
|
|
13
|
+
async retrieve(path, options) {
|
|
14
|
+
const { data, metadata } = await this.storage.get(path, { signal: options.signal });
|
|
15
|
+
if (metadata.contentEncoding != null) {
|
|
16
|
+
throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
|
|
17
|
+
}
|
|
18
|
+
if (metadata.contentType !== 'application/bson') {
|
|
19
|
+
throw new Error(`Unexpected content type: ${metadata.contentType}`);
|
|
20
|
+
}
|
|
21
|
+
const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
|
|
22
|
+
return wrapper.ops;
|
|
23
|
+
}
|
|
24
|
+
async delete(paths) {
|
|
25
|
+
return this.storage.delete(paths);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Load offloaded operations and patch them onto their MongoDB metadata documents.
|
|
30
|
+
* The object-storage implementation is responsible for limiting request
|
|
31
|
+
* concurrency across all callers.
|
|
32
|
+
*/
|
|
33
|
+
export async function hydrateBucketDataDocuments(documents, objectStorage, options) {
|
|
34
|
+
if (!objectStorage) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
options.signal?.throwIfAborted();
|
|
38
|
+
const store = new BucketDataObjectStorage(objectStorage);
|
|
39
|
+
const storedDocuments = documents.filter((document) => document.storage_ref);
|
|
40
|
+
await Promise.all(storedDocuments.map(async (document) => {
|
|
41
|
+
document.ops = await store.retrieve(document.storage_ref.path, { signal: options.signal });
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=BucketDataObjectStorage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BucketDataObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAI7B,MAAM,OAAO,uBAAuB;IACL;IAA7B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,GAAsB;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACrG,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,OAAiC;QAC5D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpF,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,GAAG,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiC,EACjC,aAAwC,EACxC,OAAiC;IAEjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IACzD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7E,MAAM,OAAO,CAAC,GAAG,CACf,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QACrC,QAAQ,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface ObjectStoragePutMetadata {
|
|
2
|
+
contentType: string;
|
|
3
|
+
contentEncoding: string | null;
|
|
4
|
+
}
|
|
5
|
+
export declare class ObjectStorageError extends Error {
|
|
6
|
+
readonly retryable: boolean;
|
|
7
|
+
constructor(message: string, options: {
|
|
8
|
+
cause: unknown;
|
|
9
|
+
retryable: boolean;
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
export declare function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError;
|
|
13
|
+
export interface ObjectStorage {
|
|
14
|
+
put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
|
|
15
|
+
get(path: string, options?: {
|
|
16
|
+
signal?: AbortSignal;
|
|
17
|
+
}): Promise<{
|
|
18
|
+
data: Uint8Array;
|
|
19
|
+
metadata: ObjectStoragePutMetadata;
|
|
20
|
+
}>;
|
|
21
|
+
list(prefix: string, options?: {
|
|
22
|
+
signal?: AbortSignal;
|
|
23
|
+
}): AsyncIterable<string>;
|
|
24
|
+
delete(paths: string[]): Promise<void>;
|
|
25
|
+
deletePrefix(prefix: string, options?: {
|
|
26
|
+
signal?: AbortSignal;
|
|
27
|
+
}): Promise<{
|
|
28
|
+
objectCount: number;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export class ObjectStorageError extends Error {
|
|
2
|
+
retryable;
|
|
3
|
+
constructor(message, options) {
|
|
4
|
+
super(message, { cause: options.cause });
|
|
5
|
+
this.name = 'ObjectStorageError';
|
|
6
|
+
this.retryable = options.retryable;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export function isRetryableObjectStorageError(error) {
|
|
10
|
+
return error instanceof ObjectStorageError && error.retryable;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=ObjectStorage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorage.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAClC,SAAS,CAAU;IAE5B,YAAY,OAAe,EAAE,OAA+C;QAC1E,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAC1D,OAAO,KAAK,YAAY,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { Logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
|
|
5
|
+
import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
|
|
6
|
+
import { ObjectStorage } from './ObjectStorage.js';
|
|
7
|
+
export declare const OBJECT_STORAGE_UPLOAD_LEASE_MS: number;
|
|
8
|
+
export declare const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS: number;
|
|
9
|
+
export declare const OBJECT_STORAGE_REFERENCE_GRACE_MS: number;
|
|
10
|
+
export interface PreparedObjectStorageUpload {
|
|
11
|
+
markerId: bson.ObjectId;
|
|
12
|
+
path: string;
|
|
13
|
+
deleteAfter: Date;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Coordinates the MongoDB outbox used to make object-storage changes recoverable.
|
|
17
|
+
*
|
|
18
|
+
* A MongoDB collection is used to track files that will be uploaded, or should be deleted.
|
|
19
|
+
* By using a MongoDB collection, we can use the same transaction that creates or removes
|
|
20
|
+
* references to these files, making the operations atomic.
|
|
21
|
+
*
|
|
22
|
+
* These markers indicate which files should be deleted - either by being orphaned during create,
|
|
23
|
+
* or after references to them are removed.
|
|
24
|
+
*/
|
|
25
|
+
export declare class ObjectStorageLifecycle {
|
|
26
|
+
private readonly db;
|
|
27
|
+
private readonly replicationStreamId;
|
|
28
|
+
private readonly objectStorage;
|
|
29
|
+
readonly bucketData: BucketDataObjectStorage;
|
|
30
|
+
constructor(db: VersionedPowerSyncMongoV3, replicationStreamId: number, objectStorage: ObjectStorage);
|
|
31
|
+
streamPrefix(): string;
|
|
32
|
+
definitionPrefix(definitionId: string): string;
|
|
33
|
+
allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string;
|
|
34
|
+
/**
|
|
35
|
+
* Indicate that files will be uploaded to S3.
|
|
36
|
+
*
|
|
37
|
+
* These markers are expected to be removed using publishUploads(). If that does not happen within
|
|
38
|
+
* the grace period, the files will be deleted.
|
|
39
|
+
*/
|
|
40
|
+
prepareUploads(paths: string[], now?: Date): Promise<PreparedObjectStorageUpload[]>;
|
|
41
|
+
canPublish(upload: PreparedObjectStorageUpload, now?: Date): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Remove markers for files that have finished uploading, and are now referenced from other documents.
|
|
44
|
+
*
|
|
45
|
+
* Call this in the same transaction as the one creating the references.
|
|
46
|
+
*/
|
|
47
|
+
publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Mark old paths for deletion.
|
|
50
|
+
*
|
|
51
|
+
* By marking for deletion instead of immediately deleting:
|
|
52
|
+
* 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
|
|
53
|
+
* be persisted transactionally, while the deletes happen asynchronously and can be retried.
|
|
54
|
+
* 2. The deletes are delayed, giving clients a grace period to continue using the files for
|
|
55
|
+
* in-progress requests.
|
|
56
|
+
*
|
|
57
|
+
* Call this in the same transaction as the one removing the references to these files.
|
|
58
|
+
*/
|
|
59
|
+
retire(paths: Iterable<string>, session: mongo.ClientSession, now?: Date): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Discover and delete objects by their storage prefix
|
|
62
|
+
* without touching the database.
|
|
63
|
+
*/
|
|
64
|
+
deletePrefix(prefix: string, options?: {
|
|
65
|
+
signal?: AbortSignal;
|
|
66
|
+
}): Promise<{
|
|
67
|
+
objectCount: number;
|
|
68
|
+
}>;
|
|
69
|
+
cleanup(logger: Logger): Promise<void>;
|
|
70
|
+
}
|