@powersync/service-core-tests 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/test-utils/storage-combinations.d.ts +13 -0
- package/dist/test-utils/storage-combinations.js +26 -0
- package/dist/test-utils/storage-combinations.js.map +1 -0
- package/dist/test-utils/test-utils-index.d.ts +1 -0
- package/dist/test-utils/test-utils-index.js +1 -0
- package/dist/test-utils/test-utils-index.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +8 -8
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +39 -14
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +3 -2
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +3 -2
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +8 -7
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/dist/tests/util.d.ts +6 -0
- package/dist/tests/util.js +12 -0
- package/dist/tests/util.js.map +1 -1
- package/package.json +4 -4
- package/src/test-utils/storage-combinations.ts +51 -0
- package/src/test-utils/test-utils-index.ts +1 -0
- package/src/tests/register-compacting-tests.ts +8 -8
- package/src/tests/register-data-storage-checkpoint-tests.ts +78 -50
- package/src/tests/register-data-storage-data-tests.ts +3 -2
- package/src/tests/register-parameter-compacting-tests.ts +3 -2
- package/src/tests/register-sync-tests.ts +8 -7
- package/src/tests/util.ts +13 -0
- package/tsconfig.tsbuildinfo +1 -1
package/dist/tests/util.d.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { storage } from '@powersync/service-core';
|
|
2
2
|
import { ParameterIndexLookupCreator, ParameterLookupScope } from '@powersync/service-sync-rules';
|
|
3
|
+
/**
|
|
4
|
+
* Resolve storage again after test writes activate the sync config. The storage
|
|
5
|
+
* instance used by the writer retains its original PROCESSING stream snapshot,
|
|
6
|
+
* while compact() expects an instance constructed with the current state.
|
|
7
|
+
*/
|
|
8
|
+
export declare function compactActive(factory: storage.BucketStorageFactory, options: storage.CompactOptions): Promise<void>;
|
|
3
9
|
export declare function bucketRequestMap(syncRules: storage.PersistedSyncConfigContent, buckets: Iterable<readonly [string, bigint]>): storage.BucketDataRequest[];
|
|
4
10
|
export declare function bucketRequests(syncRules: storage.PersistedSyncConfigContent, bucketNames: string[]): storage.BucketChecksumRequest[];
|
|
5
11
|
export declare function parameterLookupScope(lookupName: string, queryId: string, source?: ParameterIndexLookupCreator): ParameterLookupScope;
|
package/dist/tests/util.js
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
import { bucketRequest } from '../test-utils/general-utils.js';
|
|
2
|
+
/**
|
|
3
|
+
* Resolve storage again after test writes activate the sync config. The storage
|
|
4
|
+
* instance used by the writer retains its original PROCESSING stream snapshot,
|
|
5
|
+
* while compact() expects an instance constructed with the current state.
|
|
6
|
+
*/
|
|
7
|
+
export async function compactActive(factory, options) {
|
|
8
|
+
const active = await factory.getActiveSyncConfig();
|
|
9
|
+
if (active == null) {
|
|
10
|
+
throw new Error('Expected an active sync config before compacting');
|
|
11
|
+
}
|
|
12
|
+
await active.storage.compact(options);
|
|
13
|
+
}
|
|
2
14
|
export function bucketRequestMap(syncRules, buckets) {
|
|
3
15
|
return Array.from(buckets, ([bucketName, opId]) => bucketRequest(syncRules, bucketName, opId));
|
|
4
16
|
}
|
package/dist/tests/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/tests/util.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,MAAM,UAAU,gBAAgB,CAC9B,SAA6C,EAC7C,OAA4C;IAE5C,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,SAA6C,EAC7C,WAAqB;IAErB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QACzD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,mBAAmB,GAAgC;IACvD,IAAI,QAAQ;QACV,OAAO;YACL,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,GAAG;SACb,CAAC;IACJ,CAAC;IACD,eAAe;QACb,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;IACD,eAAe,CAAC,KAAK;QACnB,OAAO;YACL,oBAAoB,CAAC,WAAW,EAAE,GAAG;gBACnC,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,CAAC;IACJ,CAAC;IACD,oBAAoB,CAAC,MAAsB;QACzC,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAClC,UAAkB,EAClB,OAAe,EACf,SAAsC,mBAAmB;IAEzD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/tests/util.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqC,EAAE,OAA+B;IACxG,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;IACnD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,SAA6C,EAC7C,OAA4C;IAE5C,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,SAA6C,EAC7C,WAAqB;IAErB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QACzD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,mBAAmB,GAAgC;IACvD,IAAI,QAAQ;QACV,OAAO;YACL,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,GAAG;SACb,CAAC;IACJ,CAAC;IACD,eAAe;QACb,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;IACD,eAAe,CAAC,KAAK;QACnB,OAAO;YACL,oBAAoB,CAAC,WAAW,EAAE,GAAG;gBACnC,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,CAAC;IACJ,CAAC;IACD,oBAAoB,CAAC,MAAsB;QACzC,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAClC,UAAkB,EAClB,OAAe,EACf,SAAsC,mBAAmB;IAEzD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC"}
|
package/package.json
CHANGED
|
@@ -4,15 +4,15 @@
|
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
7
|
-
"version": "0.18.
|
|
7
|
+
"version": "0.18.2",
|
|
8
8
|
"exports": "./dist/index.js",
|
|
9
9
|
"license": "FSL-1.1-ALv2",
|
|
10
10
|
"type": "module",
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@powersync/lib-services-framework": "^0.10.
|
|
13
|
-
"@powersync/service-core": "^1.
|
|
12
|
+
"@powersync/lib-services-framework": "^0.10.2",
|
|
13
|
+
"@powersync/service-core": "^1.26.1",
|
|
14
14
|
"@powersync/service-jsonbig": "^0.17.13",
|
|
15
|
-
"@powersync/service-sync-rules": "^0.
|
|
15
|
+
"@powersync/service-sync-rules": "^0.42.0"
|
|
16
16
|
},
|
|
17
17
|
"peerDependencies": {
|
|
18
18
|
"vitest": "^4.1.5"
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import {
|
|
2
|
+
STORAGE_VERSION_1,
|
|
3
|
+
STORAGE_VERSION_2,
|
|
4
|
+
STORAGE_VERSION_4,
|
|
5
|
+
TestStorageConfig,
|
|
6
|
+
TestStorageFactory
|
|
7
|
+
} from '@powersync/service-core';
|
|
8
|
+
import { describe, TestOptions } from 'vitest';
|
|
9
|
+
|
|
10
|
+
const MONGO_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_4];
|
|
11
|
+
const POSTGRES_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2];
|
|
12
|
+
|
|
13
|
+
export interface StorageVersionTestContext {
|
|
14
|
+
factory: TestStorageFactory;
|
|
15
|
+
storageVersion: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface StorageCombinationTestConfig {
|
|
19
|
+
mongodb?: TestStorageConfig;
|
|
20
|
+
postgres?: TestStorageConfig;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function describeStorageCombinations(
|
|
24
|
+
config: StorageCombinationTestConfig,
|
|
25
|
+
options: TestOptions & { storageVersions?: number[] },
|
|
26
|
+
fn: (context: StorageVersionTestContext) => void
|
|
27
|
+
) {
|
|
28
|
+
const describeFactory = (storageName: string, storage: TestStorageConfig, supportedStorageVersions: number[]) => {
|
|
29
|
+
const storageVersions = (options.storageVersions ?? supportedStorageVersions).filter((version) =>
|
|
30
|
+
supportedStorageVersions.includes(version)
|
|
31
|
+
);
|
|
32
|
+
describe(`${storageName} storage`, options, function () {
|
|
33
|
+
for (const storageVersion of storageVersions) {
|
|
34
|
+
describe(`storage v${storageVersion}`, function () {
|
|
35
|
+
fn({
|
|
36
|
+
factory: storage.factory,
|
|
37
|
+
storageVersion
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
if (config.mongodb) {
|
|
45
|
+
describeFactory('mongodb', config.mongodb, MONGO_STORAGE_VERSIONS);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (config.postgres) {
|
|
49
|
+
describeFactory('postgres', config.postgres, POSTGRES_STORAGE_VERSIONS);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -2,5 +2,6 @@ export * from './AbstractStreamTestContext.js';
|
|
|
2
2
|
export * from './bucket-validation.js';
|
|
3
3
|
export * from './general-utils.js';
|
|
4
4
|
export * from './MetricsHelper.js';
|
|
5
|
+
export * from './storage-combinations.js';
|
|
5
6
|
export * from './StorageDataHelpers.js';
|
|
6
7
|
export * from './stream_utils.js';
|
|
@@ -2,7 +2,7 @@ import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/servi
|
|
|
2
2
|
import { expect, test } from 'vitest';
|
|
3
3
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
4
4
|
import { bucketRequest } from '../test-utils/test-utils-index.js';
|
|
5
|
-
import { bucketRequestMap, bucketRequests } from './util.js';
|
|
5
|
+
import { bucketRequestMap, bucketRequests, compactActive } from './util.js';
|
|
6
6
|
|
|
7
7
|
export function registerCompactTests(config: storage.TestStorageConfig) {
|
|
8
8
|
const generateStorageFactory = config.factory;
|
|
@@ -81,7 +81,7 @@ bucket_definitions:
|
|
|
81
81
|
]);
|
|
82
82
|
expect(batchBefore.targetOp).toEqual(null);
|
|
83
83
|
|
|
84
|
-
await
|
|
84
|
+
await compactActive(factory, {
|
|
85
85
|
clearBatchLimit: 2,
|
|
86
86
|
moveBatchLimit: 1,
|
|
87
87
|
moveBatchQueryLimit: 1,
|
|
@@ -202,7 +202,7 @@ bucket_definitions:
|
|
|
202
202
|
}
|
|
203
203
|
]);
|
|
204
204
|
|
|
205
|
-
await
|
|
205
|
+
await compactActive(factory, {
|
|
206
206
|
clearBatchLimit: 2,
|
|
207
207
|
moveBatchLimit: 1,
|
|
208
208
|
moveBatchQueryLimit: 1,
|
|
@@ -301,7 +301,7 @@ bucket_definitions:
|
|
|
301
301
|
await writer2.flush();
|
|
302
302
|
const checkpoint2 = writer2.last_flushed_op!;
|
|
303
303
|
|
|
304
|
-
await
|
|
304
|
+
await compactActive(factory, {
|
|
305
305
|
clearBatchLimit: 2,
|
|
306
306
|
moveBatchLimit: 1,
|
|
307
307
|
moveBatchQueryLimit: 1,
|
|
@@ -416,7 +416,7 @@ bucket_definitions:
|
|
|
416
416
|
|
|
417
417
|
const checkpoint = writer.last_flushed_op!;
|
|
418
418
|
|
|
419
|
-
await
|
|
419
|
+
await compactActive(factory, {
|
|
420
420
|
clearBatchLimit: 100,
|
|
421
421
|
moveBatchLimit: 100,
|
|
422
422
|
moveBatchQueryLimit: 100, // Larger limit for a larger window of operations
|
|
@@ -507,7 +507,7 @@ bucket_definitions:
|
|
|
507
507
|
await writer.commit('1/1');
|
|
508
508
|
await writer.flush();
|
|
509
509
|
|
|
510
|
-
await
|
|
510
|
+
await compactActive(factory, {
|
|
511
511
|
clearBatchLimit: 2,
|
|
512
512
|
moveBatchLimit: 1,
|
|
513
513
|
moveBatchQueryLimit: 1,
|
|
@@ -595,7 +595,7 @@ bucket_definitions:
|
|
|
595
595
|
await writer2.commit('2/1');
|
|
596
596
|
await writer2.flush();
|
|
597
597
|
|
|
598
|
-
await
|
|
598
|
+
await compactActive(factory, {
|
|
599
599
|
clearBatchLimit: 20,
|
|
600
600
|
moveBatchLimit: 10,
|
|
601
601
|
moveBatchQueryLimit: 10,
|
|
@@ -658,7 +658,7 @@ bucket_definitions:
|
|
|
658
658
|
expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1);
|
|
659
659
|
|
|
660
660
|
// With default options, Postgres compaction should use the active checkpoint.
|
|
661
|
-
await
|
|
661
|
+
await compactActive(factory, {
|
|
662
662
|
moveBatchLimit: 1,
|
|
663
663
|
moveBatchQueryLimit: 1,
|
|
664
664
|
minBucketChanges: 1,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
2
2
|
import { expect, test } from 'vitest';
|
|
3
3
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
4
|
+
import { compactActive } from './util.js';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* @example
|
|
@@ -36,7 +37,10 @@ bucket_definitions:
|
|
|
36
37
|
const abortController = new AbortController();
|
|
37
38
|
context.onTestFinished(() => abortController.abort());
|
|
38
39
|
const iter = bucketStorage
|
|
39
|
-
.watchCheckpointChanges({
|
|
40
|
+
.watchCheckpointChanges({
|
|
41
|
+
user_id: 'user1',
|
|
42
|
+
signal: abortController.signal
|
|
43
|
+
})
|
|
40
44
|
[Symbol.asyncIterator]();
|
|
41
45
|
|
|
42
46
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -85,7 +89,10 @@ bucket_definitions:
|
|
|
85
89
|
const abortController = new AbortController();
|
|
86
90
|
context.onTestFinished(() => abortController.abort());
|
|
87
91
|
const iter = bucketStorage
|
|
88
|
-
.watchCheckpointChanges({
|
|
92
|
+
.watchCheckpointChanges({
|
|
93
|
+
user_id: 'user1',
|
|
94
|
+
signal: abortController.signal
|
|
95
|
+
})
|
|
89
96
|
[Symbol.asyncIterator]();
|
|
90
97
|
|
|
91
98
|
await writer.keepalive('5/0');
|
|
@@ -194,7 +201,7 @@ bucket_definitions:
|
|
|
194
201
|
expect(generated.shouldAdvance).toBe(true);
|
|
195
202
|
});
|
|
196
203
|
|
|
197
|
-
test('custom write checkpoints - checkpoint after write', async (context) => {
|
|
204
|
+
test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - checkpoint after write', async (context) => {
|
|
198
205
|
await using factory = await generateStorageFactory();
|
|
199
206
|
const r = await factory.configureSyncRules(
|
|
200
207
|
updateSyncRulesFromYaml(
|
|
@@ -210,7 +217,9 @@ bucket_definitions:
|
|
|
210
217
|
)
|
|
211
218
|
);
|
|
212
219
|
const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
|
|
213
|
-
bucketStorage.setWriteCheckpointMode(
|
|
220
|
+
bucketStorage.setWriteCheckpointMode({
|
|
221
|
+
mode: storage.WriteCheckpointMode.CUSTOM
|
|
222
|
+
});
|
|
214
223
|
|
|
215
224
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
216
225
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -218,7 +227,10 @@ bucket_definitions:
|
|
|
218
227
|
const abortController = new AbortController();
|
|
219
228
|
context.onTestFinished(() => abortController.abort());
|
|
220
229
|
const iter = bucketStorage
|
|
221
|
-
.watchCheckpointChanges({
|
|
230
|
+
.watchCheckpointChanges({
|
|
231
|
+
user_id: 'user1',
|
|
232
|
+
signal: abortController.signal
|
|
233
|
+
})
|
|
222
234
|
[Symbol.asyncIterator]();
|
|
223
235
|
|
|
224
236
|
writer.addCustomWriteCheckpoint({
|
|
@@ -240,7 +252,7 @@ bucket_definitions:
|
|
|
240
252
|
});
|
|
241
253
|
});
|
|
242
254
|
|
|
243
|
-
test('custom write checkpoints - standalone checkpoint', async (context) => {
|
|
255
|
+
test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - standalone checkpoint', async (context) => {
|
|
244
256
|
await using factory = await generateStorageFactory();
|
|
245
257
|
const r = await factory.configureSyncRules(
|
|
246
258
|
updateSyncRulesFromYaml(
|
|
@@ -256,7 +268,9 @@ bucket_definitions:
|
|
|
256
268
|
)
|
|
257
269
|
);
|
|
258
270
|
const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
|
|
259
|
-
bucketStorage.setWriteCheckpointMode(
|
|
271
|
+
bucketStorage.setWriteCheckpointMode({
|
|
272
|
+
mode: storage.WriteCheckpointMode.CUSTOM
|
|
273
|
+
});
|
|
260
274
|
|
|
261
275
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
262
276
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -264,7 +278,10 @@ bucket_definitions:
|
|
|
264
278
|
const abortController = new AbortController();
|
|
265
279
|
context.onTestFinished(() => abortController.abort());
|
|
266
280
|
const iter = bucketStorage
|
|
267
|
-
.watchCheckpointChanges({
|
|
281
|
+
.watchCheckpointChanges({
|
|
282
|
+
user_id: 'user1',
|
|
283
|
+
signal: abortController.signal
|
|
284
|
+
})
|
|
268
285
|
[Symbol.asyncIterator]();
|
|
269
286
|
|
|
270
287
|
// Flush to clear state
|
|
@@ -289,7 +306,7 @@ bucket_definitions:
|
|
|
289
306
|
});
|
|
290
307
|
});
|
|
291
308
|
|
|
292
|
-
test('custom write checkpoints - write after checkpoint', async (context) => {
|
|
309
|
+
test.runIf((storageVersion ?? 1) < 3)('custom write checkpoints - write after checkpoint', async (context) => {
|
|
293
310
|
await using factory = await generateStorageFactory();
|
|
294
311
|
const r = await factory.configureSyncRules(
|
|
295
312
|
updateSyncRulesFromYaml(
|
|
@@ -305,7 +322,9 @@ bucket_definitions:
|
|
|
305
322
|
)
|
|
306
323
|
);
|
|
307
324
|
const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
|
|
308
|
-
bucketStorage.setWriteCheckpointMode(
|
|
325
|
+
bucketStorage.setWriteCheckpointMode({
|
|
326
|
+
mode: storage.WriteCheckpointMode.CUSTOM
|
|
327
|
+
});
|
|
309
328
|
|
|
310
329
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
311
330
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -313,7 +332,10 @@ bucket_definitions:
|
|
|
313
332
|
const abortController = new AbortController();
|
|
314
333
|
context.onTestFinished(() => abortController.abort());
|
|
315
334
|
const iter = bucketStorage
|
|
316
|
-
.watchCheckpointChanges({
|
|
335
|
+
.watchCheckpointChanges({
|
|
336
|
+
user_id: 'user1',
|
|
337
|
+
signal: abortController.signal
|
|
338
|
+
})
|
|
317
339
|
[Symbol.asyncIterator]();
|
|
318
340
|
|
|
319
341
|
await writer.keepalive('5/0');
|
|
@@ -368,49 +390,55 @@ bucket_definitions:
|
|
|
368
390
|
});
|
|
369
391
|
});
|
|
370
392
|
|
|
371
|
-
test(
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
393
|
+
test.runIf((storageVersion ?? 1) < 3)(
|
|
394
|
+
'custom write checkpoints - checkpoint request markers are temporary',
|
|
395
|
+
async () => {
|
|
396
|
+
await using factory = await generateStorageFactory();
|
|
397
|
+
const r = await factory.configureSyncRules(
|
|
398
|
+
updateSyncRulesFromYaml(
|
|
399
|
+
`
|
|
376
400
|
bucket_definitions:
|
|
377
401
|
mybucket:
|
|
378
402
|
data: []
|
|
379
403
|
`,
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
404
|
+
{
|
|
405
|
+
validate: false,
|
|
406
|
+
storageVersion
|
|
407
|
+
}
|
|
408
|
+
)
|
|
409
|
+
);
|
|
410
|
+
const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
|
|
411
|
+
bucketStorage.setWriteCheckpointMode({
|
|
412
|
+
mode: storage.WriteCheckpointMode.CUSTOM
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
416
|
+
await writer.markAllSnapshotDone('1/1');
|
|
417
|
+
|
|
418
|
+
writer.addCustomWriteCheckpoint({
|
|
419
|
+
checkpoint: 5n,
|
|
420
|
+
user_id: 'persistent'
|
|
421
|
+
});
|
|
422
|
+
writer.addCustomWriteCheckpoint({
|
|
423
|
+
checkpoint: 6n,
|
|
424
|
+
user_id: 'temporary',
|
|
425
|
+
checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z')
|
|
426
|
+
});
|
|
427
|
+
await writer.flush();
|
|
428
|
+
await writer.keepalive('1/1');
|
|
429
|
+
|
|
430
|
+
await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
|
|
431
|
+
await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n);
|
|
432
|
+
|
|
433
|
+
await compactActive(factory, {
|
|
434
|
+
compactBuckets: [],
|
|
435
|
+
deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000)
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
|
|
439
|
+
await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toBeNull();
|
|
440
|
+
}
|
|
441
|
+
);
|
|
414
442
|
}
|
|
415
443
|
|
|
416
444
|
async function createManagedWriteCheckpoint(
|
|
@@ -1458,10 +1458,11 @@ bucket_definitions:
|
|
|
1458
1458
|
test('empty storage metrics', async () => {
|
|
1459
1459
|
await using f = await generateStorageFactory({ dropAll: true });
|
|
1460
1460
|
const metrics = await f.getStorageMetrics();
|
|
1461
|
-
expect(metrics).
|
|
1461
|
+
expect(metrics).toMatchObject({
|
|
1462
1462
|
operations_size_bytes: 0,
|
|
1463
1463
|
parameters_size_bytes: 0,
|
|
1464
|
-
replication_size_bytes: 0
|
|
1464
|
+
replication_size_bytes: 0,
|
|
1465
|
+
object_storage_size_bytes: 0
|
|
1465
1466
|
});
|
|
1466
1467
|
|
|
1467
1468
|
const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
|
|
@@ -2,6 +2,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
|
2
2
|
import { ScopedParameterLookup } from '@powersync/service-sync-rules';
|
|
3
3
|
import { expect, test } from 'vitest';
|
|
4
4
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
5
|
+
import { compactActive } from './util.js';
|
|
5
6
|
|
|
6
7
|
export function registerParameterCompactTests(config: storage.TestStorageConfig) {
|
|
7
8
|
const generateStorageFactory = config.factory;
|
|
@@ -74,7 +75,7 @@ bucket_definitions:
|
|
|
74
75
|
expect(parameters2).toEqual([]);
|
|
75
76
|
|
|
76
77
|
const statsBefore = await bucketStorage.factory.getStorageMetrics();
|
|
77
|
-
await
|
|
78
|
+
await compactActive(factory, { compactParameterData: true });
|
|
78
79
|
|
|
79
80
|
// Check consistency
|
|
80
81
|
const parameters1b = await checkpoint1.getParameterSets([lookup], 1000);
|
|
@@ -154,7 +155,7 @@ bucket_definitions:
|
|
|
154
155
|
expect(parameters1).toEqual([]);
|
|
155
156
|
|
|
156
157
|
const statsBefore = await bucketStorage.factory.getStorageMetrics();
|
|
157
|
-
await
|
|
158
|
+
await compactActive(factory, { compactParameterData: true, compactParameterCacheLimit: cacheLimit });
|
|
158
159
|
|
|
159
160
|
// Check consistency
|
|
160
161
|
const parameters1b = await checkpoint1.getParameterSets([lookup], 1000);
|
|
@@ -15,6 +15,7 @@ import { fileURLToPath } from 'url';
|
|
|
15
15
|
import { expect, test, vi } from 'vitest';
|
|
16
16
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
17
17
|
import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js';
|
|
18
|
+
import { compactActive } from './util.js';
|
|
18
19
|
|
|
19
20
|
const __filename = fileURLToPath(import.meta.url);
|
|
20
21
|
const __dirname = path.dirname(__filename);
|
|
@@ -1308,6 +1309,7 @@ bucket_definitions:
|
|
|
1308
1309
|
const bucketStorage = await f.getInstance(syncRules);
|
|
1309
1310
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1310
1311
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1312
|
+
const bucket = bucketRequest(syncRules.syncConfigContent[0], 'mybucket[]').bucket;
|
|
1311
1313
|
|
|
1312
1314
|
await writer.markAllSnapshotDone('0/1');
|
|
1313
1315
|
await writer.save({
|
|
@@ -1386,10 +1388,7 @@ bucket_definitions:
|
|
|
1386
1388
|
|
|
1387
1389
|
await writer.commit('0/2');
|
|
1388
1390
|
|
|
1389
|
-
await
|
|
1390
|
-
minBucketChanges: 1,
|
|
1391
|
-
minChangeRatio: 0
|
|
1392
|
-
});
|
|
1391
|
+
await compactActive(f, { compactBuckets: [bucket] });
|
|
1393
1392
|
|
|
1394
1393
|
const lines2 = await getCheckpointLines(iter, { consume: true });
|
|
1395
1394
|
|
|
@@ -1470,6 +1469,7 @@ bucket_definitions:
|
|
|
1470
1469
|
const bucketStorage = await f.getInstance(syncRules);
|
|
1471
1470
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1472
1471
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1472
|
+
const highPriorityBucket = bucketRequest(syncRules.syncConfigContent[0], 'high_priority[]').bucket;
|
|
1473
1473
|
|
|
1474
1474
|
await writer.markAllSnapshotDone('0/1');
|
|
1475
1475
|
await writer.save({
|
|
@@ -1552,9 +1552,10 @@ bucket_definitions:
|
|
|
1552
1552
|
});
|
|
1553
1553
|
await writer.commit('0/2');
|
|
1554
1554
|
|
|
1555
|
-
await
|
|
1556
|
-
|
|
1557
|
-
|
|
1555
|
+
await compactActive(f, {
|
|
1556
|
+
// Explicitly compact the high-priority bucket: V3 schedules background
|
|
1557
|
+
// compaction, while this test needs compaction at this exact point.
|
|
1558
|
+
compactBuckets: [highPriorityBucket]
|
|
1558
1559
|
});
|
|
1559
1560
|
|
|
1560
1561
|
const lines = await getCheckpointLines(iter, { consume: true });
|
package/src/tests/util.ts
CHANGED
|
@@ -8,6 +8,19 @@ import {
|
|
|
8
8
|
} from '@powersync/service-sync-rules';
|
|
9
9
|
import { bucketRequest } from '../test-utils/general-utils.js';
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Resolve storage again after test writes activate the sync config. The storage
|
|
13
|
+
* instance used by the writer retains its original PROCESSING stream snapshot,
|
|
14
|
+
* while compact() expects an instance constructed with the current state.
|
|
15
|
+
*/
|
|
16
|
+
export async function compactActive(factory: storage.BucketStorageFactory, options: storage.CompactOptions) {
|
|
17
|
+
const active = await factory.getActiveSyncConfig();
|
|
18
|
+
if (active == null) {
|
|
19
|
+
throw new Error('Expected an active sync config before compacting');
|
|
20
|
+
}
|
|
21
|
+
await active.storage.compact(options);
|
|
22
|
+
}
|
|
23
|
+
|
|
11
24
|
export function bucketRequestMap(
|
|
12
25
|
syncRules: storage.PersistedSyncConfigContent,
|
|
13
26
|
buckets: Iterable<readonly [string, bigint]>
|