@powersync/service-core-tests 0.15.2 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/tests/register-compacting-tests.js +491 -627
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +192 -304
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +1105 -1444
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +452 -636
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-migration-tests.js +63 -139
- package/dist/tests/register-migration-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +94 -170
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +899 -1155
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +5 -5
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,55 +1,3 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
54
2
|
import { expect, test } from 'vitest';
|
|
55
3
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
@@ -58,309 +6,271 @@ import { bucketRequestMap, bucketRequests } from './util.js';
|
|
|
58
6
|
export function registerCompactTests(config) {
|
|
59
7
|
const generateStorageFactory = config.factory;
|
|
60
8
|
test('compacting (1)', async () => {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const factory = __addDisposableResource(env_1, await generateStorageFactory(), true);
|
|
64
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
9
|
+
await using factory = await generateStorageFactory();
|
|
10
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
65
11
|
bucket_definitions:
|
|
66
12
|
global:
|
|
67
13
|
data: [select * from test]
|
|
68
14
|
`));
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
-
catch (e_1) {
|
|
154
|
-
env_1.error = e_1;
|
|
155
|
-
env_1.hasError = true;
|
|
156
|
-
}
|
|
157
|
-
finally {
|
|
158
|
-
const result_1 = __disposeResources(env_1);
|
|
159
|
-
if (result_1)
|
|
160
|
-
await result_1;
|
|
161
|
-
}
|
|
15
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
16
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
17
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
18
|
+
await writer.markAllSnapshotDone('1/1');
|
|
19
|
+
await writer.save({
|
|
20
|
+
sourceTable: testTable,
|
|
21
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
22
|
+
after: {
|
|
23
|
+
id: 't1'
|
|
24
|
+
},
|
|
25
|
+
afterReplicaId: test_utils.rid('t1')
|
|
26
|
+
});
|
|
27
|
+
await writer.save({
|
|
28
|
+
sourceTable: testTable,
|
|
29
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
30
|
+
after: {
|
|
31
|
+
id: 't2'
|
|
32
|
+
},
|
|
33
|
+
afterReplicaId: test_utils.rid('t2')
|
|
34
|
+
});
|
|
35
|
+
await writer.save({
|
|
36
|
+
sourceTable: testTable,
|
|
37
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
38
|
+
after: {
|
|
39
|
+
id: 't2'
|
|
40
|
+
},
|
|
41
|
+
afterReplicaId: test_utils.rid('t2')
|
|
42
|
+
});
|
|
43
|
+
await writer.commit('1/1');
|
|
44
|
+
await writer.flush();
|
|
45
|
+
const checkpoint = writer.last_flushed_op;
|
|
46
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
47
|
+
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
48
|
+
const dataBefore = batchBefore.chunkData.data;
|
|
49
|
+
const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
50
|
+
expect(dataBefore).toMatchObject([
|
|
51
|
+
{
|
|
52
|
+
object_id: 't1',
|
|
53
|
+
op: 'PUT',
|
|
54
|
+
op_id: '1'
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
object_id: 't2',
|
|
58
|
+
op: 'PUT',
|
|
59
|
+
op_id: '2'
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
object_id: 't2',
|
|
63
|
+
op: 'PUT',
|
|
64
|
+
op_id: '3'
|
|
65
|
+
}
|
|
66
|
+
]);
|
|
67
|
+
expect(batchBefore.targetOp).toEqual(null);
|
|
68
|
+
await bucketStorage.compact({
|
|
69
|
+
clearBatchLimit: 2,
|
|
70
|
+
moveBatchLimit: 1,
|
|
71
|
+
moveBatchQueryLimit: 1,
|
|
72
|
+
minBucketChanges: 1,
|
|
73
|
+
minChangeRatio: 0
|
|
74
|
+
});
|
|
75
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
76
|
+
const dataAfter = batchAfter.chunkData.data;
|
|
77
|
+
const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
78
|
+
bucketStorage.clearChecksumCache();
|
|
79
|
+
const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
80
|
+
expect(batchAfter.targetOp).toEqual(3n);
|
|
81
|
+
expect(dataAfter).toMatchObject([
|
|
82
|
+
dataBefore[0],
|
|
83
|
+
{
|
|
84
|
+
checksum: dataBefore[1].checksum,
|
|
85
|
+
op: 'MOVE',
|
|
86
|
+
op_id: '2'
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
checksum: dataBefore[2].checksum,
|
|
90
|
+
object_id: 't2',
|
|
91
|
+
op: 'PUT',
|
|
92
|
+
op_id: '3'
|
|
93
|
+
}
|
|
94
|
+
]);
|
|
95
|
+
expect(checksumAfter.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
|
|
96
|
+
expect(checksumAfter2.get(request.bucket)).toEqual(checksumBefore.get(request.bucket));
|
|
97
|
+
test_utils.validateCompactedBucket(dataBefore, dataAfter);
|
|
162
98
|
});
|
|
163
99
|
test('compacting (2)', async () => {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
|
|
167
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
100
|
+
await using factory = await generateStorageFactory();
|
|
101
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
168
102
|
bucket_definitions:
|
|
169
103
|
global:
|
|
170
104
|
data: [select * from test]
|
|
171
105
|
`));
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
}
|
|
263
|
-
catch (e_2) {
|
|
264
|
-
env_2.error = e_2;
|
|
265
|
-
env_2.hasError = true;
|
|
266
|
-
}
|
|
267
|
-
finally {
|
|
268
|
-
const result_2 = __disposeResources(env_2);
|
|
269
|
-
if (result_2)
|
|
270
|
-
await result_2;
|
|
271
|
-
}
|
|
106
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
107
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
108
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
109
|
+
await writer.markAllSnapshotDone('1/1');
|
|
110
|
+
await writer.save({
|
|
111
|
+
sourceTable: testTable,
|
|
112
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
113
|
+
after: {
|
|
114
|
+
id: 't1'
|
|
115
|
+
},
|
|
116
|
+
afterReplicaId: test_utils.rid('t1')
|
|
117
|
+
});
|
|
118
|
+
await writer.save({
|
|
119
|
+
sourceTable: testTable,
|
|
120
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
121
|
+
after: {
|
|
122
|
+
id: 't2'
|
|
123
|
+
},
|
|
124
|
+
afterReplicaId: test_utils.rid('t2')
|
|
125
|
+
});
|
|
126
|
+
await writer.save({
|
|
127
|
+
sourceTable: testTable,
|
|
128
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
129
|
+
before: {
|
|
130
|
+
id: 't1'
|
|
131
|
+
},
|
|
132
|
+
beforeReplicaId: test_utils.rid('t1')
|
|
133
|
+
});
|
|
134
|
+
await writer.save({
|
|
135
|
+
sourceTable: testTable,
|
|
136
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
137
|
+
after: {
|
|
138
|
+
id: 't2'
|
|
139
|
+
},
|
|
140
|
+
afterReplicaId: test_utils.rid('t2')
|
|
141
|
+
});
|
|
142
|
+
await writer.commit('1/1');
|
|
143
|
+
await writer.flush();
|
|
144
|
+
const checkpoint = writer.last_flushed_op;
|
|
145
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
146
|
+
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
147
|
+
const dataBefore = batchBefore.chunkData.data;
|
|
148
|
+
const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
149
|
+
// op_id sequence depends on the storage implementation
|
|
150
|
+
expect(dataBefore).toMatchObject([
|
|
151
|
+
{
|
|
152
|
+
object_id: 't1',
|
|
153
|
+
op: 'PUT'
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
object_id: 't2',
|
|
157
|
+
op: 'PUT'
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
object_id: 't1',
|
|
161
|
+
op: 'REMOVE'
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
object_id: 't2',
|
|
165
|
+
op: 'PUT'
|
|
166
|
+
}
|
|
167
|
+
]);
|
|
168
|
+
await bucketStorage.compact({
|
|
169
|
+
clearBatchLimit: 2,
|
|
170
|
+
moveBatchLimit: 1,
|
|
171
|
+
moveBatchQueryLimit: 1,
|
|
172
|
+
minBucketChanges: 1,
|
|
173
|
+
minChangeRatio: 0
|
|
174
|
+
});
|
|
175
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
176
|
+
const dataAfter = batchAfter.chunkData.data;
|
|
177
|
+
bucketStorage.clearChecksumCache();
|
|
178
|
+
const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
179
|
+
expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
|
|
180
|
+
expect(dataAfter).toMatchObject([
|
|
181
|
+
{
|
|
182
|
+
checksum: addChecksums(addChecksums(dataBefore[0].checksum, dataBefore[1].checksum), dataBefore[2].checksum),
|
|
183
|
+
op: 'CLEAR'
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
checksum: dataBefore[3].checksum,
|
|
187
|
+
object_id: 't2',
|
|
188
|
+
op: 'PUT'
|
|
189
|
+
}
|
|
190
|
+
]);
|
|
191
|
+
expect(checksumAfter.get(request.bucket)).toEqual({
|
|
192
|
+
...checksumBefore.get(request.bucket),
|
|
193
|
+
count: 2
|
|
194
|
+
});
|
|
195
|
+
test_utils.validateCompactedBucket(dataBefore, dataAfter);
|
|
272
196
|
});
|
|
273
197
|
test('compacting (3)', async () => {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
|
|
277
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
198
|
+
await using factory = await generateStorageFactory();
|
|
199
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
278
200
|
bucket_definitions:
|
|
279
201
|
global:
|
|
280
202
|
data: [select * from test]
|
|
281
203
|
`));
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
}
|
|
349
|
-
catch (e_3) {
|
|
350
|
-
env_3.error = e_3;
|
|
351
|
-
env_3.hasError = true;
|
|
352
|
-
}
|
|
353
|
-
finally {
|
|
354
|
-
const result_3 = __disposeResources(env_3);
|
|
355
|
-
if (result_3)
|
|
356
|
-
await result_3;
|
|
357
|
-
}
|
|
204
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
205
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
206
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
207
|
+
await writer.markAllSnapshotDone('1/1');
|
|
208
|
+
await writer.save({
|
|
209
|
+
sourceTable: testTable,
|
|
210
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
211
|
+
after: {
|
|
212
|
+
id: 't1'
|
|
213
|
+
},
|
|
214
|
+
afterReplicaId: 't1'
|
|
215
|
+
});
|
|
216
|
+
await writer.save({
|
|
217
|
+
sourceTable: testTable,
|
|
218
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
219
|
+
after: {
|
|
220
|
+
id: 't2'
|
|
221
|
+
},
|
|
222
|
+
afterReplicaId: 't2'
|
|
223
|
+
});
|
|
224
|
+
await writer.save({
|
|
225
|
+
sourceTable: testTable,
|
|
226
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
227
|
+
before: {
|
|
228
|
+
id: 't1'
|
|
229
|
+
},
|
|
230
|
+
beforeReplicaId: 't1'
|
|
231
|
+
});
|
|
232
|
+
await writer.commit('1/1');
|
|
233
|
+
await writer.flush();
|
|
234
|
+
const checkpoint1 = writer.last_flushed_op;
|
|
235
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
236
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
237
|
+
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
238
|
+
await writer2.save({
|
|
239
|
+
sourceTable: testTable2,
|
|
240
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
241
|
+
before: {
|
|
242
|
+
id: 't2'
|
|
243
|
+
},
|
|
244
|
+
beforeReplicaId: 't2'
|
|
245
|
+
});
|
|
246
|
+
await writer2.commit('2/1');
|
|
247
|
+
await writer2.flush();
|
|
248
|
+
const checkpoint2 = writer2.last_flushed_op;
|
|
249
|
+
await bucketStorage.compact({
|
|
250
|
+
clearBatchLimit: 2,
|
|
251
|
+
moveBatchLimit: 1,
|
|
252
|
+
moveBatchQueryLimit: 1,
|
|
253
|
+
minBucketChanges: 1,
|
|
254
|
+
minChangeRatio: 0
|
|
255
|
+
});
|
|
256
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
257
|
+
const dataAfter = batchAfter.chunkData.data;
|
|
258
|
+
await bucketStorage.clearChecksumCache();
|
|
259
|
+
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
260
|
+
expect(dataAfter).toMatchObject([
|
|
261
|
+
{
|
|
262
|
+
op: 'CLEAR'
|
|
263
|
+
}
|
|
264
|
+
]);
|
|
265
|
+
expect(checksumAfter.get(request.bucket)).toEqual({
|
|
266
|
+
bucket: request.bucket,
|
|
267
|
+
count: 1,
|
|
268
|
+
checksum: dataAfter[0].checksum
|
|
269
|
+
});
|
|
358
270
|
});
|
|
359
271
|
test('compacting (4)', async () => {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
|
|
363
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(` bucket_definitions:
|
|
272
|
+
await using factory = await generateStorageFactory();
|
|
273
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(` bucket_definitions:
|
|
364
274
|
grouped:
|
|
365
275
|
# The parameter query here is not important
|
|
366
276
|
# We specifically don't want to create bucket_parameter records here
|
|
@@ -368,330 +278,284 @@ bucket_definitions:
|
|
|
368
278
|
parameters: select 'b' as b
|
|
369
279
|
data:
|
|
370
280
|
- select * from test where b = bucket.b`));
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
await writer.save({
|
|
384
|
-
sourceTable: testTable,
|
|
385
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
386
|
-
after: {
|
|
387
|
-
id: 't1',
|
|
388
|
-
b: 'b1',
|
|
389
|
-
value: 'start'
|
|
390
|
-
},
|
|
391
|
-
afterReplicaId: test_utils.rid('t1')
|
|
392
|
-
});
|
|
393
|
-
await writer.save({
|
|
394
|
-
sourceTable: testTable,
|
|
395
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
396
|
-
after: {
|
|
397
|
-
id: 't1',
|
|
398
|
-
b: 'b1',
|
|
399
|
-
value: 'intermediate'
|
|
400
|
-
},
|
|
401
|
-
afterReplicaId: test_utils.rid('t1')
|
|
402
|
-
});
|
|
403
|
-
await writer.save({
|
|
404
|
-
sourceTable: testTable,
|
|
405
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
406
|
-
after: {
|
|
407
|
-
id: 't2',
|
|
408
|
-
b: 'b2',
|
|
409
|
-
value: 'start'
|
|
410
|
-
},
|
|
411
|
-
afterReplicaId: test_utils.rid('t2')
|
|
412
|
-
});
|
|
413
|
-
await writer.save({
|
|
414
|
-
sourceTable: testTable,
|
|
415
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
416
|
-
after: {
|
|
417
|
-
id: 't1',
|
|
418
|
-
b: 'b1',
|
|
419
|
-
value: 'final'
|
|
420
|
-
},
|
|
421
|
-
afterReplicaId: test_utils.rid('t1')
|
|
422
|
-
});
|
|
423
|
-
await writer.save({
|
|
424
|
-
sourceTable: testTable,
|
|
425
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
426
|
-
after: {
|
|
427
|
-
id: 't2',
|
|
428
|
-
b: 'b2',
|
|
429
|
-
value: 'final'
|
|
430
|
-
},
|
|
431
|
-
afterReplicaId: test_utils.rid('t2')
|
|
432
|
-
});
|
|
433
|
-
await writer.commit('1/1');
|
|
434
|
-
}
|
|
435
|
-
await writer.flush();
|
|
436
|
-
const checkpoint = writer.last_flushed_op;
|
|
437
|
-
await bucketStorage.compact({
|
|
438
|
-
clearBatchLimit: 100,
|
|
439
|
-
moveBatchLimit: 100,
|
|
440
|
-
moveBatchQueryLimit: 100, // Larger limit for a larger window of operations
|
|
441
|
-
minBucketChanges: 1,
|
|
442
|
-
minChangeRatio: 0
|
|
443
|
-
});
|
|
444
|
-
const batchAfter = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [
|
|
445
|
-
['grouped["b1"]', 0n],
|
|
446
|
-
['grouped["b2"]', 0n]
|
|
447
|
-
])));
|
|
448
|
-
const dataAfter = batchAfter.flatMap((b) => b.chunkData.data);
|
|
449
|
-
// The op_ids will vary between MongoDB and Postgres storage
|
|
450
|
-
expect(dataAfter).toMatchObject(expect.arrayContaining([
|
|
451
|
-
{ op_id: '497', op: 'CLEAR', checksum: -937074151 },
|
|
452
|
-
{
|
|
453
|
-
op_id: '499',
|
|
454
|
-
op: 'PUT',
|
|
455
|
-
object_type: 'test',
|
|
456
|
-
object_id: 't1',
|
|
457
|
-
checksum: 52221819,
|
|
458
|
-
subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
|
|
459
|
-
data: '{"id":"t1","b":"b1","value":"final"}'
|
|
460
|
-
},
|
|
461
|
-
{ op_id: '498', op: 'CLEAR', checksum: -234380197 },
|
|
462
|
-
{
|
|
463
|
-
op_id: '500',
|
|
464
|
-
op: 'PUT',
|
|
465
|
-
object_type: 'test',
|
|
466
|
-
object_id: 't2',
|
|
467
|
-
checksum: 2126669493,
|
|
468
|
-
subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
|
|
469
|
-
data: '{"id":"t2","b":"b2","value":"final"}'
|
|
470
|
-
}
|
|
471
|
-
]));
|
|
472
|
-
}
|
|
473
|
-
catch (e_4) {
|
|
474
|
-
env_4.error = e_4;
|
|
475
|
-
env_4.hasError = true;
|
|
476
|
-
}
|
|
477
|
-
finally {
|
|
478
|
-
const result_4 = __disposeResources(env_4);
|
|
479
|
-
if (result_4)
|
|
480
|
-
await result_4;
|
|
481
|
-
}
|
|
482
|
-
});
|
|
483
|
-
test('partial checksums after compacting', async () => {
|
|
484
|
-
const env_5 = { stack: [], error: void 0, hasError: false };
|
|
485
|
-
try {
|
|
486
|
-
const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
|
|
487
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
488
|
-
bucket_definitions:
|
|
489
|
-
global:
|
|
490
|
-
data: [select * from test]
|
|
491
|
-
`));
|
|
492
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
493
|
-
const writer = __addDisposableResource(env_5, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
494
|
-
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
495
|
-
await writer.markAllSnapshotDone('1/1');
|
|
281
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
282
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
283
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
284
|
+
await writer.markAllSnapshotDone('1/1');
|
|
285
|
+
/**
|
|
286
|
+
* Repeatedly create operations which fall into different buckets.
|
|
287
|
+
* The bucket operations are purposely interleaved as the op_id increases.
|
|
288
|
+
* A large amount of operations are created here.
|
|
289
|
+
* The configured window of compacting operations is 100. This means the initial window will
|
|
290
|
+
* contain operations from multiple buckets.
|
|
291
|
+
*/
|
|
292
|
+
for (let count = 0; count < 100; count++) {
|
|
496
293
|
await writer.save({
|
|
497
294
|
sourceTable: testTable,
|
|
498
295
|
tag: storage.SaveOperationTag.INSERT,
|
|
499
296
|
after: {
|
|
500
|
-
id: 't1'
|
|
297
|
+
id: 't1',
|
|
298
|
+
b: 'b1',
|
|
299
|
+
value: 'start'
|
|
501
300
|
},
|
|
502
|
-
afterReplicaId: 't1'
|
|
301
|
+
afterReplicaId: test_utils.rid('t1')
|
|
503
302
|
});
|
|
504
303
|
await writer.save({
|
|
505
304
|
sourceTable: testTable,
|
|
506
|
-
tag: storage.SaveOperationTag.
|
|
305
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
507
306
|
after: {
|
|
508
|
-
id: '
|
|
307
|
+
id: 't1',
|
|
308
|
+
b: 'b1',
|
|
309
|
+
value: 'intermediate'
|
|
509
310
|
},
|
|
510
|
-
afterReplicaId: '
|
|
311
|
+
afterReplicaId: test_utils.rid('t1')
|
|
511
312
|
});
|
|
512
313
|
await writer.save({
|
|
513
314
|
sourceTable: testTable,
|
|
514
|
-
tag: storage.SaveOperationTag.
|
|
515
|
-
|
|
516
|
-
id: '
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
});
|
|
520
|
-
await writer.commit('1/1');
|
|
521
|
-
await writer.flush();
|
|
522
|
-
await bucketStorage.compact({
|
|
523
|
-
clearBatchLimit: 2,
|
|
524
|
-
moveBatchLimit: 1,
|
|
525
|
-
moveBatchQueryLimit: 1,
|
|
526
|
-
minBucketChanges: 1,
|
|
527
|
-
minChangeRatio: 0
|
|
528
|
-
});
|
|
529
|
-
const writer2 = __addDisposableResource(env_5, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
530
|
-
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
531
|
-
await writer2.save({
|
|
532
|
-
sourceTable: testTable2,
|
|
533
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
534
|
-
before: {
|
|
535
|
-
id: 't2'
|
|
315
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
316
|
+
after: {
|
|
317
|
+
id: 't2',
|
|
318
|
+
b: 'b2',
|
|
319
|
+
value: 'start'
|
|
536
320
|
},
|
|
537
|
-
|
|
538
|
-
});
|
|
539
|
-
await writer2.commit('2/1');
|
|
540
|
-
await writer2.flush();
|
|
541
|
-
const checkpoint2 = writer2.last_flushed_op;
|
|
542
|
-
const request = bucketRequest(syncRules, 'global[]');
|
|
543
|
-
await bucketStorage.clearChecksumCache();
|
|
544
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
545
|
-
const globalChecksum = checksumAfter.get(request.bucket);
|
|
546
|
-
expect(globalChecksum).toMatchObject({
|
|
547
|
-
bucket: request.bucket,
|
|
548
|
-
count: 4
|
|
321
|
+
afterReplicaId: test_utils.rid('t2')
|
|
549
322
|
});
|
|
550
|
-
// storage-specific checksum - just check that it does not change
|
|
551
|
-
expect(globalChecksum).toMatchSnapshot();
|
|
552
|
-
}
|
|
553
|
-
catch (e_5) {
|
|
554
|
-
env_5.error = e_5;
|
|
555
|
-
env_5.hasError = true;
|
|
556
|
-
}
|
|
557
|
-
finally {
|
|
558
|
-
const result_5 = __disposeResources(env_5);
|
|
559
|
-
if (result_5)
|
|
560
|
-
await result_5;
|
|
561
|
-
}
|
|
562
|
-
});
|
|
563
|
-
test('partial checksums after compacting (2)', async () => {
|
|
564
|
-
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
565
|
-
try {
|
|
566
|
-
const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
|
|
567
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
568
|
-
bucket_definitions:
|
|
569
|
-
global:
|
|
570
|
-
data: [select * from test]
|
|
571
|
-
`));
|
|
572
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
573
|
-
const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
574
|
-
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
575
|
-
await writer.markAllSnapshotDone('1/1');
|
|
576
323
|
await writer.save({
|
|
577
324
|
sourceTable: testTable,
|
|
578
|
-
tag: storage.SaveOperationTag.
|
|
325
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
579
326
|
after: {
|
|
580
|
-
id: 't1'
|
|
327
|
+
id: 't1',
|
|
328
|
+
b: 'b1',
|
|
329
|
+
value: 'final'
|
|
581
330
|
},
|
|
582
|
-
afterReplicaId: 't1'
|
|
331
|
+
afterReplicaId: test_utils.rid('t1')
|
|
583
332
|
});
|
|
584
333
|
await writer.save({
|
|
585
334
|
sourceTable: testTable,
|
|
586
335
|
tag: storage.SaveOperationTag.UPDATE,
|
|
587
336
|
after: {
|
|
588
|
-
id: '
|
|
337
|
+
id: 't2',
|
|
338
|
+
b: 'b2',
|
|
339
|
+
value: 'final'
|
|
589
340
|
},
|
|
590
|
-
afterReplicaId: '
|
|
341
|
+
afterReplicaId: test_utils.rid('t2')
|
|
591
342
|
});
|
|
592
343
|
await writer.commit('1/1');
|
|
593
|
-
await writer.flush();
|
|
594
|
-
// Get checksums here just to populate the cache
|
|
595
|
-
await bucketStorage.getChecksums(writer.last_flushed_op, bucketRequests(syncRules, ['global[]']));
|
|
596
|
-
const writer2 = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
597
|
-
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
598
|
-
await writer2.save({
|
|
599
|
-
sourceTable: testTable2,
|
|
600
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
601
|
-
before: {
|
|
602
|
-
id: 't1'
|
|
603
|
-
},
|
|
604
|
-
beforeReplicaId: 't1'
|
|
605
|
-
});
|
|
606
|
-
await writer2.commit('2/1');
|
|
607
|
-
await writer2.flush();
|
|
608
|
-
await bucketStorage.compact({
|
|
609
|
-
clearBatchLimit: 20,
|
|
610
|
-
moveBatchLimit: 10,
|
|
611
|
-
moveBatchQueryLimit: 10,
|
|
612
|
-
minBucketChanges: 1,
|
|
613
|
-
minChangeRatio: 0
|
|
614
|
-
});
|
|
615
|
-
const checkpoint2 = writer2.last_flushed_op;
|
|
616
|
-
const request = bucketRequest(syncRules, 'global[]');
|
|
617
|
-
// Check that the checksum was correctly updated with the clear operation after having a cached checksum
|
|
618
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
619
|
-
const globalChecksum = checksumAfter.get(request.bucket);
|
|
620
|
-
expect(globalChecksum).toMatchObject({
|
|
621
|
-
bucket: request.bucket,
|
|
622
|
-
count: 1
|
|
623
|
-
});
|
|
624
|
-
// storage-specific checksum - just check that it does not change
|
|
625
|
-
expect(globalChecksum).toMatchSnapshot();
|
|
626
|
-
}
|
|
627
|
-
catch (e_6) {
|
|
628
|
-
env_6.error = e_6;
|
|
629
|
-
env_6.hasError = true;
|
|
630
|
-
}
|
|
631
|
-
finally {
|
|
632
|
-
const result_6 = __disposeResources(env_6);
|
|
633
|
-
if (result_6)
|
|
634
|
-
await result_6;
|
|
635
344
|
}
|
|
345
|
+
await writer.flush();
|
|
346
|
+
const checkpoint = writer.last_flushed_op;
|
|
347
|
+
await bucketStorage.compact({
|
|
348
|
+
clearBatchLimit: 100,
|
|
349
|
+
moveBatchLimit: 100,
|
|
350
|
+
moveBatchQueryLimit: 100, // Larger limit for a larger window of operations
|
|
351
|
+
minBucketChanges: 1,
|
|
352
|
+
minChangeRatio: 0
|
|
353
|
+
});
|
|
354
|
+
const batchAfter = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [
|
|
355
|
+
['grouped["b1"]', 0n],
|
|
356
|
+
['grouped["b2"]', 0n]
|
|
357
|
+
])));
|
|
358
|
+
const dataAfter = batchAfter.flatMap((b) => b.chunkData.data);
|
|
359
|
+
// The op_ids will vary between MongoDB and Postgres storage
|
|
360
|
+
expect(dataAfter).toMatchObject(expect.arrayContaining([
|
|
361
|
+
{ op_id: '497', op: 'CLEAR', checksum: -937074151 },
|
|
362
|
+
{
|
|
363
|
+
op_id: '499',
|
|
364
|
+
op: 'PUT',
|
|
365
|
+
object_type: 'test',
|
|
366
|
+
object_id: 't1',
|
|
367
|
+
checksum: 52221819,
|
|
368
|
+
subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
|
|
369
|
+
data: '{"id":"t1","b":"b1","value":"final"}'
|
|
370
|
+
},
|
|
371
|
+
{ op_id: '498', op: 'CLEAR', checksum: -234380197 },
|
|
372
|
+
{
|
|
373
|
+
op_id: '500',
|
|
374
|
+
op: 'PUT',
|
|
375
|
+
object_type: 'test',
|
|
376
|
+
object_id: 't2',
|
|
377
|
+
checksum: 2126669493,
|
|
378
|
+
subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
|
|
379
|
+
data: '{"id":"t2","b":"b2","value":"final"}'
|
|
380
|
+
}
|
|
381
|
+
]));
|
|
382
|
+
});
|
|
383
|
+
test('partial checksums after compacting', async () => {
|
|
384
|
+
await using factory = await generateStorageFactory();
|
|
385
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
386
|
+
bucket_definitions:
|
|
387
|
+
global:
|
|
388
|
+
data: [select * from test]
|
|
389
|
+
`));
|
|
390
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
391
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
392
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
393
|
+
await writer.markAllSnapshotDone('1/1');
|
|
394
|
+
await writer.save({
|
|
395
|
+
sourceTable: testTable,
|
|
396
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
397
|
+
after: {
|
|
398
|
+
id: 't1'
|
|
399
|
+
},
|
|
400
|
+
afterReplicaId: 't1'
|
|
401
|
+
});
|
|
402
|
+
await writer.save({
|
|
403
|
+
sourceTable: testTable,
|
|
404
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
405
|
+
after: {
|
|
406
|
+
id: 't2'
|
|
407
|
+
},
|
|
408
|
+
afterReplicaId: 't2'
|
|
409
|
+
});
|
|
410
|
+
await writer.save({
|
|
411
|
+
sourceTable: testTable,
|
|
412
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
413
|
+
before: {
|
|
414
|
+
id: 't1'
|
|
415
|
+
},
|
|
416
|
+
beforeReplicaId: 't1'
|
|
417
|
+
});
|
|
418
|
+
await writer.commit('1/1');
|
|
419
|
+
await writer.flush();
|
|
420
|
+
await bucketStorage.compact({
|
|
421
|
+
clearBatchLimit: 2,
|
|
422
|
+
moveBatchLimit: 1,
|
|
423
|
+
moveBatchQueryLimit: 1,
|
|
424
|
+
minBucketChanges: 1,
|
|
425
|
+
minChangeRatio: 0
|
|
426
|
+
});
|
|
427
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
428
|
+
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
429
|
+
await writer2.save({
|
|
430
|
+
sourceTable: testTable2,
|
|
431
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
432
|
+
before: {
|
|
433
|
+
id: 't2'
|
|
434
|
+
},
|
|
435
|
+
beforeReplicaId: 't2'
|
|
436
|
+
});
|
|
437
|
+
await writer2.commit('2/1');
|
|
438
|
+
await writer2.flush();
|
|
439
|
+
const checkpoint2 = writer2.last_flushed_op;
|
|
440
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
441
|
+
await bucketStorage.clearChecksumCache();
|
|
442
|
+
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
443
|
+
const globalChecksum = checksumAfter.get(request.bucket);
|
|
444
|
+
expect(globalChecksum).toMatchObject({
|
|
445
|
+
bucket: request.bucket,
|
|
446
|
+
count: 4
|
|
447
|
+
});
|
|
448
|
+
// storage-specific checksum - just check that it does not change
|
|
449
|
+
expect(globalChecksum).toMatchSnapshot();
|
|
450
|
+
});
|
|
451
|
+
test('partial checksums after compacting (2)', async () => {
|
|
452
|
+
await using factory = await generateStorageFactory();
|
|
453
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
454
|
+
bucket_definitions:
|
|
455
|
+
global:
|
|
456
|
+
data: [select * from test]
|
|
457
|
+
`));
|
|
458
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
459
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
460
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
461
|
+
await writer.markAllSnapshotDone('1/1');
|
|
462
|
+
await writer.save({
|
|
463
|
+
sourceTable: testTable,
|
|
464
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
465
|
+
after: {
|
|
466
|
+
id: 't1'
|
|
467
|
+
},
|
|
468
|
+
afterReplicaId: 't1'
|
|
469
|
+
});
|
|
470
|
+
await writer.save({
|
|
471
|
+
sourceTable: testTable,
|
|
472
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
473
|
+
after: {
|
|
474
|
+
id: 't1'
|
|
475
|
+
},
|
|
476
|
+
afterReplicaId: 't1'
|
|
477
|
+
});
|
|
478
|
+
await writer.commit('1/1');
|
|
479
|
+
await writer.flush();
|
|
480
|
+
// Get checksums here just to populate the cache
|
|
481
|
+
await bucketStorage.getChecksums(writer.last_flushed_op, bucketRequests(syncRules, ['global[]']));
|
|
482
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
483
|
+
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
484
|
+
await writer2.save({
|
|
485
|
+
sourceTable: testTable2,
|
|
486
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
487
|
+
before: {
|
|
488
|
+
id: 't1'
|
|
489
|
+
},
|
|
490
|
+
beforeReplicaId: 't1'
|
|
491
|
+
});
|
|
492
|
+
await writer2.commit('2/1');
|
|
493
|
+
await writer2.flush();
|
|
494
|
+
await bucketStorage.compact({
|
|
495
|
+
clearBatchLimit: 20,
|
|
496
|
+
moveBatchLimit: 10,
|
|
497
|
+
moveBatchQueryLimit: 10,
|
|
498
|
+
minBucketChanges: 1,
|
|
499
|
+
minChangeRatio: 0
|
|
500
|
+
});
|
|
501
|
+
const checkpoint2 = writer2.last_flushed_op;
|
|
502
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
503
|
+
// Check that the checksum was correctly updated with the clear operation after having a cached checksum
|
|
504
|
+
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
505
|
+
const globalChecksum = checksumAfter.get(request.bucket);
|
|
506
|
+
expect(globalChecksum).toMatchObject({
|
|
507
|
+
bucket: request.bucket,
|
|
508
|
+
count: 1
|
|
509
|
+
});
|
|
510
|
+
// storage-specific checksum - just check that it does not change
|
|
511
|
+
expect(globalChecksum).toMatchSnapshot();
|
|
636
512
|
});
|
|
637
513
|
test('defaults maxOpId to current checkpoint', async () => {
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
|
|
641
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
514
|
+
await using factory = await generateStorageFactory();
|
|
515
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
642
516
|
bucket_definitions:
|
|
643
517
|
global:
|
|
644
518
|
data: [select * from test]
|
|
645
519
|
`));
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
}
|
|
686
|
-
catch (e_7) {
|
|
687
|
-
env_7.error = e_7;
|
|
688
|
-
env_7.hasError = true;
|
|
689
|
-
}
|
|
690
|
-
finally {
|
|
691
|
-
const result_7 = __disposeResources(env_7);
|
|
692
|
-
if (result_7)
|
|
693
|
-
await result_7;
|
|
694
|
-
}
|
|
520
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
521
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
522
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
523
|
+
await writer.markAllSnapshotDone('1/1');
|
|
524
|
+
await writer.save({
|
|
525
|
+
sourceTable: testTable,
|
|
526
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
527
|
+
after: { id: 't1' },
|
|
528
|
+
afterReplicaId: test_utils.rid('t1')
|
|
529
|
+
});
|
|
530
|
+
await writer.commit('1/1');
|
|
531
|
+
await writer.flush();
|
|
532
|
+
const checkpoint1 = writer.last_flushed_op;
|
|
533
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
534
|
+
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
535
|
+
// This is flushed but not committed (does not advance the checkpoint)
|
|
536
|
+
await writer2.save({
|
|
537
|
+
sourceTable: testTable2,
|
|
538
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
539
|
+
after: { id: 't1' },
|
|
540
|
+
afterReplicaId: test_utils.rid('t1')
|
|
541
|
+
});
|
|
542
|
+
await writer2.flush();
|
|
543
|
+
const checkpoint2 = writer2.last_flushed_op;
|
|
544
|
+
const checkpointBeforeCompact = await bucketStorage.getCheckpoint();
|
|
545
|
+
expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1);
|
|
546
|
+
// With default options, Postgres compaction should use the active checkpoint.
|
|
547
|
+
await bucketStorage.compact({
|
|
548
|
+
moveBatchLimit: 1,
|
|
549
|
+
moveBatchQueryLimit: 1,
|
|
550
|
+
minBucketChanges: 1,
|
|
551
|
+
minChangeRatio: 0
|
|
552
|
+
});
|
|
553
|
+
const batchAfterDefaultCompact = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]])));
|
|
554
|
+
// Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.
|
|
555
|
+
expect(batchAfterDefaultCompact.chunkData.data).toMatchObject([
|
|
556
|
+
{ op_id: '1', op: 'PUT', object_id: 't1' },
|
|
557
|
+
{ op_id: '2', op: 'PUT', object_id: 't1' }
|
|
558
|
+
]);
|
|
695
559
|
});
|
|
696
560
|
}
|
|
697
561
|
//# sourceMappingURL=register-compacting-tests.js.map
|