@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
|
|
2
|
+
import { SqlSyncConfigParser, updateSyncRulesFromConfig } from '@powersync/service-core';
|
|
3
|
+
import { test_utils } from '@powersync/service-core-tests';
|
|
4
|
+
import {
|
|
5
|
+
AdditionalSyncConfigParser,
|
|
6
|
+
normalizeConnectionConfig,
|
|
7
|
+
PrecompiledSyncConfig
|
|
8
|
+
} from '@powersync/service-sync-rules';
|
|
9
|
+
import { describe, expect, test } from 'vitest';
|
|
10
|
+
import { env } from './env.js';
|
|
11
|
+
import { TEST_STORAGE_VERSIONS } from './util.js';
|
|
12
|
+
|
|
13
|
+
const STREAMS = /* yaml */ `
|
|
14
|
+
config:
|
|
15
|
+
edition: 3
|
|
16
|
+
streams:
|
|
17
|
+
orders:
|
|
18
|
+
query: SELECT * FROM orders
|
|
19
|
+
`;
|
|
20
|
+
const CONFIGURED = /* yaml */ `
|
|
21
|
+
config:
|
|
22
|
+
edition: 3
|
|
23
|
+
connections:
|
|
24
|
+
default:
|
|
25
|
+
type: example
|
|
26
|
+
tables:
|
|
27
|
+
orders:
|
|
28
|
+
sample: 10
|
|
29
|
+
streams:
|
|
30
|
+
orders:
|
|
31
|
+
query: SELECT * FROM orders
|
|
32
|
+
`;
|
|
33
|
+
|
|
34
|
+
// A small module-owned option exercises persistence without introducing MongoDB filter syntax into core.
|
|
35
|
+
const TABLE_OPTIONS: AdditionalSyncConfigParser = {
|
|
36
|
+
id: 'example.tables',
|
|
37
|
+
extendJsonSchema({ schema }) {
|
|
38
|
+
const map = (schema.properties as any).config.properties.connections;
|
|
39
|
+
map.additionalProperties = {
|
|
40
|
+
if: { type: 'object', required: ['type'], properties: { type: { const: 'example' } } },
|
|
41
|
+
then: {
|
|
42
|
+
type: 'object',
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
required: ['type'],
|
|
45
|
+
properties: {
|
|
46
|
+
type: { const: 'example' },
|
|
47
|
+
tables: {
|
|
48
|
+
type: 'object',
|
|
49
|
+
additionalProperties: {
|
|
50
|
+
type: 'object',
|
|
51
|
+
additionalProperties: false,
|
|
52
|
+
properties: { sample: { type: 'number' } }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
else: map.additionalProperties
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
parse({ config, context }) {
|
|
61
|
+
const connections = normalizeConnectionConfig(
|
|
62
|
+
(config as { config?: { connections?: unknown } }).config?.connections
|
|
63
|
+
);
|
|
64
|
+
for (const [tag, connection] of Object.entries(connections)) {
|
|
65
|
+
if (connection?.type != 'example') continue;
|
|
66
|
+
context.parsedConfig.connectionConfig = { ...context.parsedConfig.connectionConfig, [tag]: connection };
|
|
67
|
+
const { plan } = context.parsedConfig as PrecompiledSyncConfig;
|
|
68
|
+
plan.moduleData = { ...plan.moduleData, ['example.tables']: null };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
function storageFactory(withModule: boolean) {
|
|
74
|
+
const syncConfigParser = new SqlSyncConfigParser(withModule ? [TABLE_OPTIONS] : []);
|
|
75
|
+
return mongoTestStorageFactoryGenerator({
|
|
76
|
+
url: env.MONGO_TEST_URL,
|
|
77
|
+
isCI: env.CI,
|
|
78
|
+
supportsMultipleSyncConfigs: true,
|
|
79
|
+
syncConfigParser
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe.for(TEST_STORAGE_VERSIONS)('connection config with storage v%i', (storageVersion) => {
|
|
84
|
+
test('preserves module options across closing storage and loading its compiled plan', async () => {
|
|
85
|
+
let expected;
|
|
86
|
+
{
|
|
87
|
+
await using factory = await storageFactory(true).factory();
|
|
88
|
+
const parsed = factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS);
|
|
89
|
+
expected = parsed.config.connectionConfig;
|
|
90
|
+
await factory.updateSyncRules(updateSyncRulesFromConfig(parsed, { storageVersion }));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Reopen the same database without clearing it, as a service restart would.
|
|
94
|
+
await using reopened = await storageFactory(true).factory({ doNotClear: true });
|
|
95
|
+
const deploying = (await reopened.getDeployingSyncConfig())!;
|
|
96
|
+
expect(deploying.content.compiled_plan?.plan.version).toBe(3);
|
|
97
|
+
const parsed = deploying.content.parsed(test_utils.PARSE_OPTIONS);
|
|
98
|
+
expect(parsed.syncConfigs[0].config.connectionConfig).toEqual(expected);
|
|
99
|
+
expect(parsed.hydratedSyncConfig.connectionConfig).toEqual(expected);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('rejects saved options if the module is absent after restart', async () => {
|
|
103
|
+
{
|
|
104
|
+
await using factory = await storageFactory(true).factory();
|
|
105
|
+
await factory.updateSyncRules(
|
|
106
|
+
updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS), {
|
|
107
|
+
storageVersion
|
|
108
|
+
})
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
await using reopened = await storageFactory(false).factory({ doNotClear: true });
|
|
112
|
+
const deploying = (await reopened.getDeployingSyncConfig())!;
|
|
113
|
+
expect(() => deploying.content.parsed(test_utils.PARSE_OPTIONS)).toThrow(
|
|
114
|
+
'Missing required sync config parsers: example.tables'
|
|
115
|
+
);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test('continues loading unfiltered plans without an additional parser', async () => {
|
|
119
|
+
await using factory = await storageFactory(false).factory();
|
|
120
|
+
await factory.updateSyncRules(
|
|
121
|
+
updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(STREAMS, test_utils.PARSE_OPTIONS), {
|
|
122
|
+
storageVersion
|
|
123
|
+
})
|
|
124
|
+
);
|
|
125
|
+
const deploying = (await factory.getDeployingSyncConfig())!;
|
|
126
|
+
expect(deploying.content.compiled_plan?.plan.version).toBe(1);
|
|
127
|
+
expect(deploying.content.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig.connectionConfig).toEqual({});
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test('starts replacement processing when connection options change', async () => {
|
|
131
|
+
await using factory = await storageFactory(true).factory();
|
|
132
|
+
const deploy = (yaml: string) =>
|
|
133
|
+
factory.updateSyncRules(
|
|
134
|
+
updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(yaml, test_utils.PARSE_OPTIONS), {
|
|
135
|
+
storageVersion
|
|
136
|
+
})
|
|
137
|
+
);
|
|
138
|
+
const first = await deploy(CONFIGURED);
|
|
139
|
+
{
|
|
140
|
+
// Activate the first config so compatibility is checked against data already being served.
|
|
141
|
+
const storage = await test_utils.getTestStorage(factory, first);
|
|
142
|
+
await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
143
|
+
await writer.markAllSnapshotDone('1/1');
|
|
144
|
+
await writer.commit('1/1');
|
|
145
|
+
}
|
|
146
|
+
if (storageVersion >= 3) {
|
|
147
|
+
// SQL-only edits can share source data when the connection options are identical.
|
|
148
|
+
const sameOptions = await deploy(CONFIGURED.replace('SELECT * FROM orders', 'SELECT id FROM orders'));
|
|
149
|
+
expect(sameOptions.replicationStreamId).toBe(first.replicationStreamId);
|
|
150
|
+
const storage = await test_utils.getTestStorage(factory, sameOptions);
|
|
151
|
+
await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
152
|
+
await writer.markAllSnapshotDone('2/1');
|
|
153
|
+
await writer.commit('2/1');
|
|
154
|
+
}
|
|
155
|
+
const changed = await deploy(CONFIGURED.replace('sample: 10', 'sample: 20'));
|
|
156
|
+
expect(changed.replicationStreamId).not.toBe(first.replicationStreamId);
|
|
157
|
+
expect((await factory.getActiveSyncConfig())?.content.replicationStreamId).toBe(first.replicationStreamId);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ObjectStorage,
|
|
3
|
+
ObjectStorageOperationOptions,
|
|
3
4
|
ObjectStoragePutMetadata
|
|
4
5
|
} from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
|
|
5
6
|
|
|
@@ -9,13 +10,19 @@ export class MemoryObjectStorage implements ObjectStorage {
|
|
|
9
10
|
*/
|
|
10
11
|
public readonly store = new Map<string, { data: Uint8Array; metadata: ObjectStoragePutMetadata }>();
|
|
11
12
|
|
|
12
|
-
async put(
|
|
13
|
+
async put(
|
|
14
|
+
path: string,
|
|
15
|
+
data: Uint8Array,
|
|
16
|
+
metadata: ObjectStoragePutMetadata,
|
|
17
|
+
options?: ObjectStorageOperationOptions
|
|
18
|
+
): Promise<void> {
|
|
19
|
+
options?.signal?.throwIfAborted();
|
|
13
20
|
this.store.set(path, { data, metadata: metadata });
|
|
14
21
|
}
|
|
15
22
|
|
|
16
23
|
async get(
|
|
17
24
|
path: string,
|
|
18
|
-
options?:
|
|
25
|
+
options?: ObjectStorageOperationOptions
|
|
19
26
|
): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
|
|
20
27
|
const signal = options?.signal;
|
|
21
28
|
signal?.throwIfAborted();
|
|
@@ -26,7 +33,7 @@ export class MemoryObjectStorage implements ObjectStorage {
|
|
|
26
33
|
return data;
|
|
27
34
|
}
|
|
28
35
|
|
|
29
|
-
async *list(prefix: string, options?:
|
|
36
|
+
async *list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string> {
|
|
30
37
|
const signal = options?.signal;
|
|
31
38
|
for (const path of this.store.keys()) {
|
|
32
39
|
signal?.throwIfAborted();
|
|
@@ -36,18 +43,19 @@ export class MemoryObjectStorage implements ObjectStorage {
|
|
|
36
43
|
}
|
|
37
44
|
}
|
|
38
45
|
|
|
39
|
-
async delete(paths: string[]): Promise<void> {
|
|
46
|
+
async delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void> {
|
|
47
|
+
options?.signal?.throwIfAborted();
|
|
40
48
|
for (const p of paths) {
|
|
41
49
|
this.store.delete(p);
|
|
42
50
|
}
|
|
43
51
|
}
|
|
44
52
|
|
|
45
|
-
async deletePrefix(prefix: string, options?:
|
|
53
|
+
async deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{ objectCount: number }> {
|
|
46
54
|
const paths: string[] = [];
|
|
47
55
|
for await (const path of this.list(prefix, options)) {
|
|
48
56
|
paths.push(path);
|
|
49
57
|
}
|
|
50
|
-
await this.delete(paths);
|
|
58
|
+
await this.delete(paths, options);
|
|
51
59
|
return { objectCount: paths.length };
|
|
52
60
|
}
|
|
53
61
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { test_utils } from '@powersync/service-core-tests';
|
|
3
|
+
import { mongoTestStorageFactoryGenerator } from '../../../dist/utils/test-utils.js';
|
|
4
|
+
|
|
5
|
+
// A separate process exercises the database protocol without sharing any mutex,
|
|
6
|
+
// allocator, MongoClient, or lease object with the test runner.
|
|
7
|
+
async function main() {
|
|
8
|
+
const factoryGen = mongoTestStorageFactoryGenerator({ url: process.argv[2], isCI: true });
|
|
9
|
+
await using factory = await factoryGen.factory({ doNotClear: true });
|
|
10
|
+
const streamId = Number(process.argv[3]);
|
|
11
|
+
const stream = (await factory.getReplicatingReplicationStreams()).find((s) => s.replicationStreamId === streamId);
|
|
12
|
+
const lock = await stream.lock();
|
|
13
|
+
await using lockLifetime = { [Symbol.asyncDispose]: () => lock.release() };
|
|
14
|
+
const bucketStorage = factory.getInstance(stream, { replicationLock: lock });
|
|
15
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
16
|
+
const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, streamId);
|
|
17
|
+
await writer.save({
|
|
18
|
+
sourceTable: table,
|
|
19
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
20
|
+
after: { id: 'child', description: 'child' },
|
|
21
|
+
afterReplicaId: test_utils.rid('child')
|
|
22
|
+
});
|
|
23
|
+
await writer.commit('1/2');
|
|
24
|
+
console.log(`checkpoint=${(await bucketStorage.getCheckpoint()).checkpoint}`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
await main();
|
|
28
|
+
// Workspace test utilities start background timers. Exit after all resources are disposed.
|
|
29
|
+
process.exit(0);
|
|
@@ -4,7 +4,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
|
|
|
4
4
|
import { randomUUID } from 'node:crypto';
|
|
5
5
|
import { MemoryObjectStorage } from './MemoryObjectStorage.js';
|
|
6
6
|
|
|
7
|
-
const
|
|
7
|
+
const rustfsStorages: S3ObjectStorage[] = [];
|
|
8
8
|
|
|
9
9
|
export interface S3TestFactoryOptions {
|
|
10
10
|
url: string;
|
|
@@ -26,24 +26,24 @@ function createTestStorageSuite(options: S3TestFactoryOptions, objectStorage: Ob
|
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Creates an ObjectStorage instance for S3 tests.
|
|
29
|
-
* Set
|
|
30
|
-
* to a real
|
|
31
|
-
*
|
|
29
|
+
* Set RUSTFS_ENDPOINT to switch all S3 tests from MemoryObjectStorage
|
|
30
|
+
* to a real RustFS/S3 endpoint.
|
|
31
|
+
* RUSTFS_ENDPOINT=http://localhost:9000
|
|
32
32
|
*/
|
|
33
33
|
export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
|
|
34
|
-
const
|
|
34
|
+
const rustfsEndpoint = process.env.RUSTFS_ENDPOINT;
|
|
35
35
|
let objectStorage: ObjectStorage;
|
|
36
|
-
if (
|
|
36
|
+
if (rustfsEndpoint) {
|
|
37
37
|
const s3 = new S3ObjectStorage({
|
|
38
38
|
bucket: 'powersync-s3-test',
|
|
39
39
|
region: 'us-east-1',
|
|
40
40
|
prefix: `test-${process.pid}-${randomUUID()}`,
|
|
41
|
-
endpoint:
|
|
41
|
+
endpoint: rustfsEndpoint,
|
|
42
42
|
forcePathStyle: true,
|
|
43
|
-
accessKeyId: process.env.
|
|
44
|
-
secretAccessKey: process.env.
|
|
43
|
+
accessKeyId: process.env.RUSTFS_ACCESS_KEY ?? 'rustfsadmin',
|
|
44
|
+
secretAccessKey: process.env.RUSTFS_SECRET_KEY ?? 'rustfsadmin'
|
|
45
45
|
});
|
|
46
|
-
|
|
46
|
+
rustfsStorages.push(s3);
|
|
47
47
|
objectStorage = s3;
|
|
48
48
|
} else {
|
|
49
49
|
objectStorage = new MemoryObjectStorage();
|
|
@@ -52,9 +52,9 @@ export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
|
|
|
52
52
|
return createTestStorageSuite(options, objectStorage);
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
/** Remove all objects created by
|
|
55
|
+
/** Remove all objects created by RustFS-backed suites in the current test. */
|
|
56
56
|
export async function cleanupS3TestStorage(): Promise<void> {
|
|
57
|
-
for (const storage of
|
|
57
|
+
for (const storage of rustfsStorages.splice(0)) {
|
|
58
58
|
await storage.deletePrefix('bucket-data/');
|
|
59
59
|
}
|
|
60
60
|
}
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js';
|
|
2
|
+
import { BucketDataDocumentV3, ObjectStorageUsageDocument } from '@module/storage/implementation/v3/models.js';
|
|
3
|
+
import {
|
|
4
|
+
OBJECT_STORAGE_USAGE_BASE_WRITER_ID,
|
|
5
|
+
ObjectStorageUsage
|
|
6
|
+
} from '@module/storage/implementation/v3/object-storage/ObjectStorageUsage.js';
|
|
7
|
+
import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
8
|
+
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
9
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
10
|
+
import { updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
11
|
+
import { getTestStorage } from '@powersync/service-core-tests';
|
|
12
|
+
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
13
|
+
import { describe, expect, test } from 'vitest';
|
|
14
|
+
import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
|
|
15
|
+
|
|
16
|
+
const SYNC_RULES_YAML = `
|
|
17
|
+
bucket_definitions:
|
|
18
|
+
global:
|
|
19
|
+
data:
|
|
20
|
+
- SELECT id FROM items
|
|
21
|
+
`;
|
|
22
|
+
|
|
23
|
+
type UsageContext = {
|
|
24
|
+
db: VersionedPowerSyncMongoV3;
|
|
25
|
+
bucketStorage: MongoSyncBucketStorage;
|
|
26
|
+
definitionId: BucketDefinitionId;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
type UsageEntry = {
|
|
30
|
+
definition_id: BucketDefinitionId;
|
|
31
|
+
active_bytes: bigint;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
async function withUsageContext<T>(callback: (context: UsageContext) => Promise<T>): Promise<T> {
|
|
35
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
36
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
37
|
+
const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
|
|
38
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
39
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
40
|
+
return callback({ db, bucketStorage, definitionId });
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function withTransaction<T>(
|
|
44
|
+
db: VersionedPowerSyncMongoV3,
|
|
45
|
+
callback: (session: mongo.ClientSession) => Promise<T>
|
|
46
|
+
): Promise<T> {
|
|
47
|
+
const session = db.client.startSession();
|
|
48
|
+
try {
|
|
49
|
+
return await session.withTransaction(() => callback(session));
|
|
50
|
+
} finally {
|
|
51
|
+
await session.endSession();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function readUsageEntries(
|
|
56
|
+
db: VersionedPowerSyncMongoV3,
|
|
57
|
+
replicationStreamId: number,
|
|
58
|
+
session?: mongo.ClientSession
|
|
59
|
+
): Promise<UsageEntry[]> {
|
|
60
|
+
const entries = await db.objectStorageUsage
|
|
61
|
+
.aggregate<{ _id: BucketDefinitionId; active_bytes: bigint }>(
|
|
62
|
+
[
|
|
63
|
+
{ $match: { '_id.g': replicationStreamId } },
|
|
64
|
+
{ $project: { definitions: { $objectToArray: '$definitions' } } },
|
|
65
|
+
{ $unwind: '$definitions' },
|
|
66
|
+
{ $group: { _id: '$definitions.k', active_bytes: { $sum: '$definitions.v' } } }
|
|
67
|
+
],
|
|
68
|
+
{ session, readConcern: 'snapshot' }
|
|
69
|
+
)
|
|
70
|
+
.toArray()
|
|
71
|
+
.catch((error) => {
|
|
72
|
+
if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
throw error;
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
return entries.map((entry) => ({
|
|
79
|
+
definition_id: entry._id,
|
|
80
|
+
active_bytes: BigInt(entry.active_bytes ?? 0)
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function usageDocument(
|
|
85
|
+
replicationStreamId: number,
|
|
86
|
+
writerId: string,
|
|
87
|
+
definitions: Record<string, bigint>,
|
|
88
|
+
updatedAt = new Date(0)
|
|
89
|
+
): ObjectStorageUsageDocument {
|
|
90
|
+
return {
|
|
91
|
+
_id: { g: replicationStreamId, w: writerId },
|
|
92
|
+
updated_at: updatedAt,
|
|
93
|
+
definitions
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function bucketDataDocument(
|
|
98
|
+
replicationStreamId: number,
|
|
99
|
+
definitionId: BucketDefinitionId,
|
|
100
|
+
fileSize: number
|
|
101
|
+
): BucketDataDocumentV3 {
|
|
102
|
+
return {
|
|
103
|
+
_id: { b: `${replicationStreamId}-${definitionId}`, o: 1n },
|
|
104
|
+
min_op: 1n,
|
|
105
|
+
checksum: 1n,
|
|
106
|
+
count: 1,
|
|
107
|
+
size: fileSize,
|
|
108
|
+
storage_ref: {
|
|
109
|
+
path: `bucket-data/${replicationStreamId}/${definitionId}/1.bson`,
|
|
110
|
+
file_size: fileSize
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
describe('ObjectStorageUsage', () => {
|
|
116
|
+
test('counts a committed reference exactly once after an aborted transaction is retried', async () => {
|
|
117
|
+
await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
|
|
118
|
+
const replicationStreamId = bucketStorage.replicationStreamId;
|
|
119
|
+
const writerId = 'replication-writer';
|
|
120
|
+
const usage = new ObjectStorageUsage(db, replicationStreamId, writerId);
|
|
121
|
+
const bucketData = db.bucketData(replicationStreamId, definitionId);
|
|
122
|
+
const document = bucketDataDocument(replicationStreamId, definitionId, 123);
|
|
123
|
+
|
|
124
|
+
await expect(
|
|
125
|
+
withTransaction(db, async (session) => {
|
|
126
|
+
await bucketData.insertOne(document, { session });
|
|
127
|
+
const writes = db.createWriteBatch(session, { ordered: false });
|
|
128
|
+
usage.applyDelta(definitionId, 123n, writes);
|
|
129
|
+
await writes.execute();
|
|
130
|
+
throw new Error('abort this attempt');
|
|
131
|
+
})
|
|
132
|
+
).rejects.toThrow('abort this attempt');
|
|
133
|
+
|
|
134
|
+
await withTransaction(db, async (session) => {
|
|
135
|
+
await bucketData.insertOne(document, { session });
|
|
136
|
+
const writes = db.createWriteBatch(session, { ordered: false });
|
|
137
|
+
usage.applyDelta(definitionId, 123n, writes);
|
|
138
|
+
await writes.execute();
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
expect(await bucketData.countDocuments({ storage_ref: { $exists: true } })).toBe(1);
|
|
142
|
+
await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
|
|
143
|
+
{ definition_id: definitionId, active_bytes: 123n }
|
|
144
|
+
]);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('snapshot usage reads observe a complete pre-fold view', async () => {
|
|
149
|
+
await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
|
|
150
|
+
const replicationStreamId = bucketStorage.replicationStreamId;
|
|
151
|
+
const usage = new ObjectStorageUsage(db, replicationStreamId);
|
|
152
|
+
await db.objectStorageUsage.insertMany([
|
|
153
|
+
usageDocument(replicationStreamId, OBJECT_STORAGE_USAGE_BASE_WRITER_ID, { [definitionId]: 100n }),
|
|
154
|
+
usageDocument(replicationStreamId, 'replication', { [definitionId]: 20n }),
|
|
155
|
+
usageDocument(replicationStreamId, 'compactor', { [definitionId]: -5n })
|
|
156
|
+
]);
|
|
157
|
+
|
|
158
|
+
const readSession = db.client.startSession();
|
|
159
|
+
try {
|
|
160
|
+
readSession.startTransaction({ readConcern: { level: 'snapshot' } });
|
|
161
|
+
await db.objectStorageUsage.findOne(
|
|
162
|
+
{ _id: { g: replicationStreamId, w: OBJECT_STORAGE_USAGE_BASE_WRITER_ID } },
|
|
163
|
+
{ session: readSession }
|
|
164
|
+
);
|
|
165
|
+
|
|
166
|
+
await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
|
|
167
|
+
const entries = await readUsageEntries(db, replicationStreamId, readSession);
|
|
168
|
+
|
|
169
|
+
await readSession.commitTransaction();
|
|
170
|
+
expect(entries).toEqual([{ definition_id: definitionId, active_bytes: 115n }]);
|
|
171
|
+
} catch (error) {
|
|
172
|
+
await readSession.abortTransaction().catch(() => undefined);
|
|
173
|
+
throw error;
|
|
174
|
+
} finally {
|
|
175
|
+
await readSession.endSession();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
|
|
179
|
+
{ definition_id: definitionId, active_bytes: 115n }
|
|
180
|
+
]);
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test('folding preserves totals across both writer/folder commit orders', async () => {
|
|
185
|
+
await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
|
|
186
|
+
const replicationStreamId = bucketStorage.replicationStreamId;
|
|
187
|
+
const writerId = 'reused-writer';
|
|
188
|
+
const usage = new ObjectStorageUsage(db, replicationStreamId, writerId);
|
|
189
|
+
|
|
190
|
+
await withTransaction(db, async (session) => {
|
|
191
|
+
const writes = db.createWriteBatch(session, { ordered: false });
|
|
192
|
+
usage.applyDelta(definitionId, 10n, writes);
|
|
193
|
+
await writes.execute();
|
|
194
|
+
});
|
|
195
|
+
await db.objectStorageUsage.updateOne(
|
|
196
|
+
{ _id: { g: replicationStreamId, w: writerId } },
|
|
197
|
+
{ $set: { updated_at: new Date(0) } }
|
|
198
|
+
);
|
|
199
|
+
await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
|
|
200
|
+
|
|
201
|
+
// The folder committed first and deleted the writer shard. The writer's
|
|
202
|
+
// next transaction must recreate it through the upsert.
|
|
203
|
+
await withTransaction(db, async (session) => {
|
|
204
|
+
const writes = db.createWriteBatch(session, { ordered: false });
|
|
205
|
+
usage.applyDelta(definitionId, 4n, writes);
|
|
206
|
+
await writes.execute();
|
|
207
|
+
});
|
|
208
|
+
expect((await readUsageEntries(db, replicationStreamId))[0].active_bytes).toBe(14n);
|
|
209
|
+
|
|
210
|
+
await db.objectStorageUsage.updateOne(
|
|
211
|
+
{ _id: { g: replicationStreamId, w: writerId } },
|
|
212
|
+
{ $set: { updated_at: new Date(0) } }
|
|
213
|
+
);
|
|
214
|
+
await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
|
|
215
|
+
|
|
216
|
+
// The writer committed before the folder. Folding the recreated shard
|
|
217
|
+
// must preserve the same visible total.
|
|
218
|
+
expect((await readUsageEntries(db, replicationStreamId))[0].active_bytes).toBe(14n);
|
|
219
|
+
});
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test('definition cleanup racing a fold removes only that definition, and stream clear is isolated', async () => {
|
|
223
|
+
await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
|
|
224
|
+
const replicationStreamId = bucketStorage.replicationStreamId;
|
|
225
|
+
const otherStreamId = replicationStreamId + 1;
|
|
226
|
+
const otherDefinitionId = 'other_definition';
|
|
227
|
+
const usage = new ObjectStorageUsage(db, replicationStreamId);
|
|
228
|
+
|
|
229
|
+
await db.objectStorageUsage.insertMany([
|
|
230
|
+
usageDocument(replicationStreamId, OBJECT_STORAGE_USAGE_BASE_WRITER_ID, {
|
|
231
|
+
[definitionId]: 3n,
|
|
232
|
+
[otherDefinitionId]: 7n
|
|
233
|
+
}),
|
|
234
|
+
usageDocument(replicationStreamId, 'writer', { [definitionId]: 20n, [otherDefinitionId]: 11n }),
|
|
235
|
+
usageDocument(otherStreamId, 'other-writer', { [definitionId]: 50n })
|
|
236
|
+
]);
|
|
237
|
+
|
|
238
|
+
await Promise.all([
|
|
239
|
+
usage.foldStaleWriterDeltas({ staleWriterMs: 0 }),
|
|
240
|
+
withTransaction(db, (session) => usage.removeDefinition(definitionId, session))
|
|
241
|
+
]);
|
|
242
|
+
|
|
243
|
+
await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
|
|
244
|
+
{ definition_id: otherDefinitionId, active_bytes: 18n }
|
|
245
|
+
]);
|
|
246
|
+
|
|
247
|
+
await withTransaction(db, async (session) => {
|
|
248
|
+
await new ObjectStorageUsage(db, otherStreamId).removeStream(session);
|
|
249
|
+
});
|
|
250
|
+
await db.objectStorageUsage.insertOne(usageDocument(otherStreamId, 'other-writer', { [definitionId]: 50n }));
|
|
251
|
+
|
|
252
|
+
await withTransaction(db, async (session) => {
|
|
253
|
+
await usage.removeStream(session);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
await expect(ObjectStorageUsage.readAllDefinitionUsage(db)).resolves.toEqual([
|
|
257
|
+
{ replication_stream_id: otherStreamId, definition_id: definitionId, active_bytes: 50n }
|
|
258
|
+
]);
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
test('returns no entries for an absent usage collection', async () => {
|
|
263
|
+
await withUsageContext(async ({ db, bucketStorage }) => {
|
|
264
|
+
await db.objectStorageUsage.drop().catch(() => undefined);
|
|
265
|
+
await expect(readUsageEntries(db, bucketStorage.replicationStreamId)).resolves.toEqual([]);
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
});
|