@powersync/service-core-tests 0.15.1 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/tests/register-compacting-tests.js +491 -627
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +192 -304
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +1105 -1444
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +452 -636
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-migration-tests.js +63 -139
- package/dist/tests/register-migration-tests.js.map +1 -1
- package/dist/tests/register-parameter-compacting-tests.js +94 -170
- package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +899 -1155
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/package.json +5 -5
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,55 +1,3 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
1
|
import { CURRENT_STORAGE_VERSION, getUuidReplicaIdentityBson, reduceBucket, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
54
2
|
import { describe, expect, test } from 'vitest';
|
|
55
3
|
import * as test_utils from '../test-utils/test-utils-index.js';
|
|
@@ -78,372 +26,223 @@ export function registerDataStorageDataTests(config) {
|
|
|
78
26
|
const generateStorageFactory = config.factory;
|
|
79
27
|
const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
|
|
80
28
|
test('removing row', async () => {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const factory = __addDisposableResource(env_1, await generateStorageFactory(), true);
|
|
84
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
29
|
+
await using factory = await generateStorageFactory();
|
|
30
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
85
31
|
bucket_definitions:
|
|
86
32
|
global:
|
|
87
33
|
data:
|
|
88
34
|
- SELECT id, description FROM "%"
|
|
89
35
|
`, { storageVersion }));
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
}
|
|
134
|
-
catch (e_1) {
|
|
135
|
-
env_1.error = e_1;
|
|
136
|
-
env_1.hasError = true;
|
|
137
|
-
}
|
|
138
|
-
finally {
|
|
139
|
-
const result_1 = __disposeResources(env_1);
|
|
140
|
-
if (result_1)
|
|
141
|
-
await result_1;
|
|
142
|
-
}
|
|
36
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
37
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
38
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
39
|
+
await writer.markAllSnapshotDone('1/1');
|
|
40
|
+
await writer.save({
|
|
41
|
+
sourceTable: testTable,
|
|
42
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
43
|
+
after: {
|
|
44
|
+
id: 'test1',
|
|
45
|
+
description: 'test1'
|
|
46
|
+
},
|
|
47
|
+
afterReplicaId: test_utils.rid('test1')
|
|
48
|
+
});
|
|
49
|
+
await writer.save({
|
|
50
|
+
sourceTable: testTable,
|
|
51
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
52
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
53
|
+
});
|
|
54
|
+
await writer.commit('1/1');
|
|
55
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
56
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
57
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
58
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
59
|
+
return {
|
|
60
|
+
op: d.op,
|
|
61
|
+
object_id: d.object_id,
|
|
62
|
+
checksum: d.checksum
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
const c1 = 2871785649;
|
|
66
|
+
const c2 = 2872534815;
|
|
67
|
+
expect(data).toEqual([
|
|
68
|
+
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
69
|
+
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
70
|
+
]);
|
|
71
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
72
|
+
expect(checksums).toEqual([
|
|
73
|
+
{
|
|
74
|
+
bucket: request.bucket,
|
|
75
|
+
checksum: (c1 + c2) & 0xffffffff,
|
|
76
|
+
count: 2
|
|
77
|
+
}
|
|
78
|
+
]);
|
|
143
79
|
});
|
|
144
80
|
test('insert after delete in new batch', async () => {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
|
|
148
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
81
|
+
await using factory = await generateStorageFactory();
|
|
82
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
149
83
|
bucket_definitions:
|
|
150
84
|
global:
|
|
151
85
|
data:
|
|
152
86
|
- SELECT id, description FROM "%"
|
|
153
87
|
`, { storageVersion }));
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
}
|
|
195
|
-
catch (e_2) {
|
|
196
|
-
env_2.error = e_2;
|
|
197
|
-
env_2.hasError = true;
|
|
198
|
-
}
|
|
199
|
-
finally {
|
|
200
|
-
const result_2 = __disposeResources(env_2);
|
|
201
|
-
if (result_2)
|
|
202
|
-
await result_2;
|
|
203
|
-
}
|
|
88
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
89
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
90
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
91
|
+
await writer.markAllSnapshotDone('1/1');
|
|
92
|
+
await writer.save({
|
|
93
|
+
sourceTable: testTable,
|
|
94
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
95
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
96
|
+
});
|
|
97
|
+
await writer.commit('0/1');
|
|
98
|
+
await writer.save({
|
|
99
|
+
sourceTable: testTable,
|
|
100
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
101
|
+
after: {
|
|
102
|
+
id: 'test1',
|
|
103
|
+
description: 'test1'
|
|
104
|
+
},
|
|
105
|
+
afterReplicaId: test_utils.rid('test1')
|
|
106
|
+
});
|
|
107
|
+
await writer.commit('2/1');
|
|
108
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
109
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
110
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
111
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
112
|
+
return {
|
|
113
|
+
op: d.op,
|
|
114
|
+
object_id: d.object_id,
|
|
115
|
+
checksum: d.checksum
|
|
116
|
+
};
|
|
117
|
+
});
|
|
118
|
+
const c1 = 2871785649;
|
|
119
|
+
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
120
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
121
|
+
expect(checksums).toEqual([
|
|
122
|
+
{
|
|
123
|
+
bucket: request.bucket,
|
|
124
|
+
checksum: c1 & 0xffffffff,
|
|
125
|
+
count: 1
|
|
126
|
+
}
|
|
127
|
+
]);
|
|
204
128
|
});
|
|
205
129
|
test('update after delete in new batch', async () => {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
|
|
210
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
130
|
+
// Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way.
|
|
131
|
+
await using factory = await generateStorageFactory();
|
|
132
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
211
133
|
bucket_definitions:
|
|
212
134
|
global:
|
|
213
135
|
data:
|
|
214
136
|
- SELECT id, description FROM "%"
|
|
215
137
|
`, { storageVersion }));
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
}
|
|
261
|
-
catch (e_3) {
|
|
262
|
-
env_3.error = e_3;
|
|
263
|
-
env_3.hasError = true;
|
|
264
|
-
}
|
|
265
|
-
finally {
|
|
266
|
-
const result_3 = __disposeResources(env_3);
|
|
267
|
-
if (result_3)
|
|
268
|
-
await result_3;
|
|
269
|
-
}
|
|
138
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
139
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
140
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
141
|
+
await writer.markAllSnapshotDone('1/1');
|
|
142
|
+
await writer.save({
|
|
143
|
+
sourceTable: testTable,
|
|
144
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
145
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
146
|
+
});
|
|
147
|
+
await writer.commit('0/1');
|
|
148
|
+
await writer.save({
|
|
149
|
+
sourceTable: testTable,
|
|
150
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
151
|
+
before: {
|
|
152
|
+
id: 'test1'
|
|
153
|
+
},
|
|
154
|
+
after: {
|
|
155
|
+
id: 'test1',
|
|
156
|
+
description: 'test1'
|
|
157
|
+
},
|
|
158
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
159
|
+
afterReplicaId: test_utils.rid('test1')
|
|
160
|
+
});
|
|
161
|
+
await writer.commit('2/1');
|
|
162
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
163
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
164
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
165
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
166
|
+
return {
|
|
167
|
+
op: d.op,
|
|
168
|
+
object_id: d.object_id,
|
|
169
|
+
checksum: d.checksum
|
|
170
|
+
};
|
|
171
|
+
});
|
|
172
|
+
const c1 = 2871785649;
|
|
173
|
+
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
174
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
175
|
+
expect(checksums).toEqual([
|
|
176
|
+
{
|
|
177
|
+
bucket: request.bucket,
|
|
178
|
+
checksum: c1 & 0xffffffff,
|
|
179
|
+
count: 1
|
|
180
|
+
}
|
|
181
|
+
]);
|
|
270
182
|
});
|
|
271
183
|
test('insert after delete in same batch', async () => {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
|
|
275
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
184
|
+
await using factory = await generateStorageFactory();
|
|
185
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
276
186
|
bucket_definitions:
|
|
277
187
|
global:
|
|
278
188
|
data:
|
|
279
189
|
- SELECT id, description FROM "%"
|
|
280
190
|
`, {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
{
|
|
317
|
-
bucket: request.bucket,
|
|
318
|
-
checksum: c1 & 0xffffffff,
|
|
319
|
-
count: 1
|
|
320
|
-
}
|
|
321
|
-
]);
|
|
322
|
-
}
|
|
323
|
-
catch (e_4) {
|
|
324
|
-
env_4.error = e_4;
|
|
325
|
-
env_4.hasError = true;
|
|
326
|
-
}
|
|
327
|
-
finally {
|
|
328
|
-
const result_4 = __disposeResources(env_4);
|
|
329
|
-
if (result_4)
|
|
330
|
-
await result_4;
|
|
331
|
-
}
|
|
332
|
-
});
|
|
333
|
-
test('(insert, delete, insert), (delete)', async () => {
|
|
334
|
-
const env_5 = { stack: [], error: void 0, hasError: false };
|
|
335
|
-
try {
|
|
336
|
-
const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
|
|
337
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
338
|
-
bucket_definitions:
|
|
339
|
-
global:
|
|
340
|
-
data:
|
|
341
|
-
- SELECT id, description FROM "%"
|
|
342
|
-
`, {
|
|
343
|
-
storageVersion
|
|
344
|
-
}));
|
|
345
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
346
|
-
{
|
|
347
|
-
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
348
|
-
try {
|
|
349
|
-
const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
350
|
-
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
351
|
-
await writer.markAllSnapshotDone('1/1');
|
|
352
|
-
await writer.save({
|
|
353
|
-
sourceTable,
|
|
354
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
355
|
-
after: {
|
|
356
|
-
id: 'test1',
|
|
357
|
-
description: 'test1'
|
|
358
|
-
},
|
|
359
|
-
afterReplicaId: test_utils.rid('test1')
|
|
360
|
-
});
|
|
361
|
-
await writer.save({
|
|
362
|
-
sourceTable,
|
|
363
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
364
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
365
|
-
});
|
|
366
|
-
await writer.save({
|
|
367
|
-
sourceTable,
|
|
368
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
369
|
-
after: {
|
|
370
|
-
id: 'test1',
|
|
371
|
-
description: 'test1'
|
|
372
|
-
},
|
|
373
|
-
afterReplicaId: test_utils.rid('test1')
|
|
374
|
-
});
|
|
375
|
-
await writer.commit('1/1');
|
|
376
|
-
}
|
|
377
|
-
catch (e_5) {
|
|
378
|
-
env_6.error = e_5;
|
|
379
|
-
env_6.hasError = true;
|
|
380
|
-
}
|
|
381
|
-
finally {
|
|
382
|
-
const result_5 = __disposeResources(env_6);
|
|
383
|
-
if (result_5)
|
|
384
|
-
await result_5;
|
|
385
|
-
}
|
|
386
|
-
}
|
|
191
|
+
storageVersion
|
|
192
|
+
}));
|
|
193
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
194
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
195
|
+
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
196
|
+
await writer.markAllSnapshotDone('1/1');
|
|
197
|
+
await writer.save({
|
|
198
|
+
sourceTable: testTable,
|
|
199
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
200
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
201
|
+
});
|
|
202
|
+
await writer.save({
|
|
203
|
+
sourceTable: testTable,
|
|
204
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
205
|
+
after: {
|
|
206
|
+
id: 'test1',
|
|
207
|
+
description: 'test1'
|
|
208
|
+
},
|
|
209
|
+
afterReplicaId: test_utils.rid('test1')
|
|
210
|
+
});
|
|
211
|
+
await writer.commit('1/1');
|
|
212
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
213
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
214
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
215
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
216
|
+
return {
|
|
217
|
+
op: d.op,
|
|
218
|
+
object_id: d.object_id,
|
|
219
|
+
checksum: d.checksum
|
|
220
|
+
};
|
|
221
|
+
});
|
|
222
|
+
const c1 = 2871785649;
|
|
223
|
+
expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]);
|
|
224
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
225
|
+
expect(checksums).toEqual([
|
|
387
226
|
{
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
392
|
-
await writer.markAllSnapshotDone('1/1');
|
|
393
|
-
await writer.save({
|
|
394
|
-
sourceTable,
|
|
395
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
396
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
397
|
-
});
|
|
398
|
-
await writer.commit('2/1');
|
|
399
|
-
}
|
|
400
|
-
catch (e_6) {
|
|
401
|
-
env_7.error = e_6;
|
|
402
|
-
env_7.hasError = true;
|
|
403
|
-
}
|
|
404
|
-
finally {
|
|
405
|
-
const result_6 = __disposeResources(env_7);
|
|
406
|
-
if (result_6)
|
|
407
|
-
await result_6;
|
|
408
|
-
}
|
|
227
|
+
bucket: request.bucket,
|
|
228
|
+
checksum: c1 & 0xffffffff,
|
|
229
|
+
count: 1
|
|
409
230
|
}
|
|
410
|
-
|
|
411
|
-
const request = bucketRequest(syncRules, 'global[]');
|
|
412
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
413
|
-
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
414
|
-
const data = batch[0].chunkData.data.map((d) => {
|
|
415
|
-
return {
|
|
416
|
-
op: d.op,
|
|
417
|
-
object_id: d.object_id,
|
|
418
|
-
checksum: d.checksum
|
|
419
|
-
};
|
|
420
|
-
});
|
|
421
|
-
expect(data).toMatchSnapshot();
|
|
422
|
-
}
|
|
423
|
-
catch (e_7) {
|
|
424
|
-
env_5.error = e_7;
|
|
425
|
-
env_5.hasError = true;
|
|
426
|
-
}
|
|
427
|
-
finally {
|
|
428
|
-
const result_7 = __disposeResources(env_5);
|
|
429
|
-
if (result_7)
|
|
430
|
-
await result_7;
|
|
431
|
-
}
|
|
231
|
+
]);
|
|
432
232
|
});
|
|
433
|
-
test('
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
|
|
437
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
233
|
+
test('(insert, delete, insert), (delete)', async () => {
|
|
234
|
+
await using factory = await generateStorageFactory();
|
|
235
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
438
236
|
bucket_definitions:
|
|
439
237
|
global:
|
|
440
238
|
data:
|
|
441
|
-
- SELECT
|
|
239
|
+
- SELECT id, description FROM "%"
|
|
442
240
|
`, {
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
241
|
+
storageVersion
|
|
242
|
+
}));
|
|
243
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
244
|
+
{
|
|
245
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
447
246
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
448
247
|
await writer.markAllSnapshotDone('1/1');
|
|
449
248
|
await writer.save({
|
|
@@ -451,73 +250,15 @@ bucket_definitions:
|
|
|
451
250
|
tag: storage.SaveOperationTag.INSERT,
|
|
452
251
|
after: {
|
|
453
252
|
id: 'test1',
|
|
454
|
-
|
|
455
|
-
description: 'test1a'
|
|
456
|
-
},
|
|
457
|
-
afterReplicaId: test_utils.rid('test1')
|
|
458
|
-
});
|
|
459
|
-
await writer.save({
|
|
460
|
-
sourceTable,
|
|
461
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
462
|
-
after: {
|
|
463
|
-
id: 'test1',
|
|
464
|
-
client_id: 'client1b',
|
|
465
|
-
description: 'test1b'
|
|
253
|
+
description: 'test1'
|
|
466
254
|
},
|
|
467
255
|
afterReplicaId: test_utils.rid('test1')
|
|
468
256
|
});
|
|
469
257
|
await writer.save({
|
|
470
258
|
sourceTable,
|
|
471
|
-
tag: storage.SaveOperationTag.
|
|
472
|
-
|
|
473
|
-
id: 'test2',
|
|
474
|
-
client_id: 'client2',
|
|
475
|
-
description: 'test2'
|
|
476
|
-
},
|
|
477
|
-
afterReplicaId: test_utils.rid('test2')
|
|
478
|
-
});
|
|
479
|
-
await writer.commit('1/1');
|
|
480
|
-
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
481
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
482
|
-
const data = batch[0].chunkData.data.map((d) => {
|
|
483
|
-
return {
|
|
484
|
-
op: d.op,
|
|
485
|
-
object_id: d.object_id
|
|
486
|
-
};
|
|
259
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
260
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
487
261
|
});
|
|
488
|
-
expect(data).toEqual([
|
|
489
|
-
{ op: 'PUT', object_id: 'client1a' },
|
|
490
|
-
{ op: 'PUT', object_id: 'client1b' },
|
|
491
|
-
{ op: 'REMOVE', object_id: 'client1a' },
|
|
492
|
-
{ op: 'PUT', object_id: 'client2' }
|
|
493
|
-
]);
|
|
494
|
-
}
|
|
495
|
-
catch (e_8) {
|
|
496
|
-
env_8.error = e_8;
|
|
497
|
-
env_8.hasError = true;
|
|
498
|
-
}
|
|
499
|
-
finally {
|
|
500
|
-
const result_8 = __disposeResources(env_8);
|
|
501
|
-
if (result_8)
|
|
502
|
-
await result_8;
|
|
503
|
-
}
|
|
504
|
-
});
|
|
505
|
-
test('re-apply delete', async () => {
|
|
506
|
-
const env_9 = { stack: [], error: void 0, hasError: false };
|
|
507
|
-
try {
|
|
508
|
-
const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
|
|
509
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
510
|
-
bucket_definitions:
|
|
511
|
-
global:
|
|
512
|
-
data:
|
|
513
|
-
- SELECT id, description FROM "%"
|
|
514
|
-
`, {
|
|
515
|
-
storageVersion
|
|
516
|
-
}));
|
|
517
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
518
|
-
const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
519
|
-
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
520
|
-
await writer.markAllSnapshotDone('1/1');
|
|
521
262
|
await writer.save({
|
|
522
263
|
sourceTable,
|
|
523
264
|
tag: storage.SaveOperationTag.INSERT,
|
|
@@ -527,738 +268,753 @@ bucket_definitions:
|
|
|
527
268
|
},
|
|
528
269
|
afterReplicaId: test_utils.rid('test1')
|
|
529
270
|
});
|
|
530
|
-
await writer.flush();
|
|
531
|
-
await writer.save({
|
|
532
|
-
sourceTable,
|
|
533
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
534
|
-
beforeReplicaId: test_utils.rid('test1')
|
|
535
|
-
});
|
|
536
271
|
await writer.commit('1/1');
|
|
272
|
+
}
|
|
273
|
+
{
|
|
274
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
275
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
276
|
+
await writer.markAllSnapshotDone('1/1');
|
|
537
277
|
await writer.save({
|
|
538
278
|
sourceTable,
|
|
539
279
|
tag: storage.SaveOperationTag.DELETE,
|
|
540
280
|
beforeReplicaId: test_utils.rid('test1')
|
|
541
281
|
});
|
|
542
|
-
await writer.
|
|
543
|
-
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
544
|
-
const request = bucketRequest(syncRules, 'global[]');
|
|
545
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
546
|
-
const data = batch[0].chunkData.data.map((d) => {
|
|
547
|
-
return {
|
|
548
|
-
op: d.op,
|
|
549
|
-
object_id: d.object_id,
|
|
550
|
-
checksum: d.checksum
|
|
551
|
-
};
|
|
552
|
-
});
|
|
553
|
-
const c1 = 2871785649;
|
|
554
|
-
const c2 = 2872534815;
|
|
555
|
-
expect(data).toEqual([
|
|
556
|
-
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
557
|
-
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
558
|
-
]);
|
|
559
|
-
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
560
|
-
expect(checksums).toEqual([
|
|
561
|
-
{
|
|
562
|
-
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
563
|
-
checksum: (c1 + c2) & 0xffffffff,
|
|
564
|
-
count: 2
|
|
565
|
-
}
|
|
566
|
-
]);
|
|
567
|
-
}
|
|
568
|
-
catch (e_9) {
|
|
569
|
-
env_9.error = e_9;
|
|
570
|
-
env_9.hasError = true;
|
|
571
|
-
}
|
|
572
|
-
finally {
|
|
573
|
-
const result_9 = __disposeResources(env_9);
|
|
574
|
-
if (result_9)
|
|
575
|
-
await result_9;
|
|
282
|
+
await writer.commit('2/1');
|
|
576
283
|
}
|
|
284
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
285
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
286
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
287
|
+
expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]);
|
|
288
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
289
|
+
return {
|
|
290
|
+
op: d.op,
|
|
291
|
+
object_id: d.object_id,
|
|
292
|
+
checksum: d.checksum
|
|
293
|
+
};
|
|
294
|
+
});
|
|
295
|
+
expect(data).toMatchSnapshot();
|
|
296
|
+
});
|
|
297
|
+
test('changing client ids', async () => {
|
|
298
|
+
await using factory = await generateStorageFactory();
|
|
299
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
300
|
+
bucket_definitions:
|
|
301
|
+
global:
|
|
302
|
+
data:
|
|
303
|
+
- SELECT client_id as id, description FROM "%"
|
|
304
|
+
`, {
|
|
305
|
+
storageVersion
|
|
306
|
+
}));
|
|
307
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
308
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
309
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
310
|
+
await writer.markAllSnapshotDone('1/1');
|
|
311
|
+
await writer.save({
|
|
312
|
+
sourceTable,
|
|
313
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
314
|
+
after: {
|
|
315
|
+
id: 'test1',
|
|
316
|
+
client_id: 'client1a',
|
|
317
|
+
description: 'test1a'
|
|
318
|
+
},
|
|
319
|
+
afterReplicaId: test_utils.rid('test1')
|
|
320
|
+
});
|
|
321
|
+
await writer.save({
|
|
322
|
+
sourceTable,
|
|
323
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
324
|
+
after: {
|
|
325
|
+
id: 'test1',
|
|
326
|
+
client_id: 'client1b',
|
|
327
|
+
description: 'test1b'
|
|
328
|
+
},
|
|
329
|
+
afterReplicaId: test_utils.rid('test1')
|
|
330
|
+
});
|
|
331
|
+
await writer.save({
|
|
332
|
+
sourceTable,
|
|
333
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
334
|
+
after: {
|
|
335
|
+
id: 'test2',
|
|
336
|
+
client_id: 'client2',
|
|
337
|
+
description: 'test2'
|
|
338
|
+
},
|
|
339
|
+
afterReplicaId: test_utils.rid('test2')
|
|
340
|
+
});
|
|
341
|
+
await writer.commit('1/1');
|
|
342
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
343
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
344
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
345
|
+
return {
|
|
346
|
+
op: d.op,
|
|
347
|
+
object_id: d.object_id
|
|
348
|
+
};
|
|
349
|
+
});
|
|
350
|
+
expect(data).toEqual([
|
|
351
|
+
{ op: 'PUT', object_id: 'client1a' },
|
|
352
|
+
{ op: 'PUT', object_id: 'client1b' },
|
|
353
|
+
{ op: 'REMOVE', object_id: 'client1a' },
|
|
354
|
+
{ op: 'PUT', object_id: 'client2' }
|
|
355
|
+
]);
|
|
356
|
+
});
|
|
357
|
+
test('re-apply delete', async () => {
|
|
358
|
+
await using factory = await generateStorageFactory();
|
|
359
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
360
|
+
bucket_definitions:
|
|
361
|
+
global:
|
|
362
|
+
data:
|
|
363
|
+
- SELECT id, description FROM "%"
|
|
364
|
+
`, {
|
|
365
|
+
storageVersion
|
|
366
|
+
}));
|
|
367
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
368
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
369
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
370
|
+
await writer.markAllSnapshotDone('1/1');
|
|
371
|
+
await writer.save({
|
|
372
|
+
sourceTable,
|
|
373
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
374
|
+
after: {
|
|
375
|
+
id: 'test1',
|
|
376
|
+
description: 'test1'
|
|
377
|
+
},
|
|
378
|
+
afterReplicaId: test_utils.rid('test1')
|
|
379
|
+
});
|
|
380
|
+
await writer.flush();
|
|
381
|
+
await writer.save({
|
|
382
|
+
sourceTable,
|
|
383
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
384
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
385
|
+
});
|
|
386
|
+
await writer.commit('1/1');
|
|
387
|
+
await writer.save({
|
|
388
|
+
sourceTable,
|
|
389
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
390
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
391
|
+
});
|
|
392
|
+
await writer.flush();
|
|
393
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
394
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
395
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
396
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
397
|
+
return {
|
|
398
|
+
op: d.op,
|
|
399
|
+
object_id: d.object_id,
|
|
400
|
+
checksum: d.checksum
|
|
401
|
+
};
|
|
402
|
+
});
|
|
403
|
+
const c1 = 2871785649;
|
|
404
|
+
const c2 = 2872534815;
|
|
405
|
+
expect(data).toEqual([
|
|
406
|
+
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
407
|
+
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
408
|
+
]);
|
|
409
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
410
|
+
expect(checksums).toEqual([
|
|
411
|
+
{
|
|
412
|
+
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
413
|
+
checksum: (c1 + c2) & 0xffffffff,
|
|
414
|
+
count: 2
|
|
415
|
+
}
|
|
416
|
+
]);
|
|
577
417
|
});
|
|
578
418
|
test('re-apply update + delete', async () => {
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
const factory = __addDisposableResource(env_10, await generateStorageFactory(), true);
|
|
582
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
419
|
+
await using factory = await generateStorageFactory();
|
|
420
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
583
421
|
bucket_definitions:
|
|
584
422
|
global:
|
|
585
423
|
data:
|
|
586
424
|
- SELECT id, description FROM "%"
|
|
587
425
|
`, { storageVersion }));
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
}
|
|
679
|
-
catch (e_10) {
|
|
680
|
-
env_10.error = e_10;
|
|
681
|
-
env_10.hasError = true;
|
|
682
|
-
}
|
|
683
|
-
finally {
|
|
684
|
-
const result_10 = __disposeResources(env_10);
|
|
685
|
-
if (result_10)
|
|
686
|
-
await result_10;
|
|
687
|
-
}
|
|
426
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
427
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
428
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
429
|
+
await writer.markAllSnapshotDone('1/1');
|
|
430
|
+
await writer.save({
|
|
431
|
+
sourceTable,
|
|
432
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
433
|
+
after: {
|
|
434
|
+
id: 'test1',
|
|
435
|
+
description: 'test1'
|
|
436
|
+
},
|
|
437
|
+
afterReplicaId: test_utils.rid('test1')
|
|
438
|
+
});
|
|
439
|
+
await writer.flush();
|
|
440
|
+
await writer.markAllSnapshotDone('1/1');
|
|
441
|
+
await writer.save({
|
|
442
|
+
sourceTable,
|
|
443
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
444
|
+
after: {
|
|
445
|
+
id: 'test1',
|
|
446
|
+
description: undefined
|
|
447
|
+
},
|
|
448
|
+
afterReplicaId: test_utils.rid('test1')
|
|
449
|
+
});
|
|
450
|
+
await writer.save({
|
|
451
|
+
sourceTable,
|
|
452
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
453
|
+
after: {
|
|
454
|
+
id: 'test1',
|
|
455
|
+
description: undefined
|
|
456
|
+
},
|
|
457
|
+
afterReplicaId: test_utils.rid('test1')
|
|
458
|
+
});
|
|
459
|
+
await writer.save({
|
|
460
|
+
sourceTable,
|
|
461
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
462
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
463
|
+
});
|
|
464
|
+
await writer.commit('1/1');
|
|
465
|
+
await writer.markAllSnapshotDone('1/1');
|
|
466
|
+
await writer.save({
|
|
467
|
+
sourceTable,
|
|
468
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
469
|
+
after: {
|
|
470
|
+
id: 'test1',
|
|
471
|
+
description: undefined
|
|
472
|
+
},
|
|
473
|
+
afterReplicaId: test_utils.rid('test1')
|
|
474
|
+
});
|
|
475
|
+
await writer.save({
|
|
476
|
+
sourceTable,
|
|
477
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
478
|
+
after: {
|
|
479
|
+
id: 'test1',
|
|
480
|
+
description: undefined
|
|
481
|
+
},
|
|
482
|
+
afterReplicaId: test_utils.rid('test1')
|
|
483
|
+
});
|
|
484
|
+
await writer.save({
|
|
485
|
+
sourceTable,
|
|
486
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
487
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
488
|
+
});
|
|
489
|
+
await writer.commit('2/1');
|
|
490
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
491
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
492
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
493
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
494
|
+
return {
|
|
495
|
+
op: d.op,
|
|
496
|
+
object_id: d.object_id,
|
|
497
|
+
checksum: d.checksum
|
|
498
|
+
};
|
|
499
|
+
});
|
|
500
|
+
const c1 = 2871785649;
|
|
501
|
+
const c2 = 2872534815;
|
|
502
|
+
expect(data).toEqual([
|
|
503
|
+
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
504
|
+
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
505
|
+
{ op: 'PUT', object_id: 'test1', checksum: c1 },
|
|
506
|
+
{ op: 'REMOVE', object_id: 'test1', checksum: c2 }
|
|
507
|
+
]);
|
|
508
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
509
|
+
expect(checksums).toEqual([
|
|
510
|
+
{
|
|
511
|
+
bucket: bucketRequest(syncRules, 'global[]').bucket,
|
|
512
|
+
checksum: (c1 + c1 + c1 + c2) & 0xffffffff,
|
|
513
|
+
count: 4
|
|
514
|
+
}
|
|
515
|
+
]);
|
|
688
516
|
});
|
|
689
517
|
test('batch with overlapping replica ids', async () => {
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
|
|
700
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
518
|
+
// This test checks that we get the correct output when processing rows with:
|
|
519
|
+
// 1. changing replica ids
|
|
520
|
+
// 2. overlapping with replica ids of other rows in the same transaction (at different times)
|
|
521
|
+
// If operations are not processing in input order, this breaks easily.
|
|
522
|
+
// It can break at two places:
|
|
523
|
+
// 1. Not getting the correct "current_data" state for each operation.
|
|
524
|
+
// 2. Output order not being correct.
|
|
525
|
+
await using factory = await generateStorageFactory();
|
|
526
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
701
527
|
bucket_definitions:
|
|
702
528
|
global:
|
|
703
529
|
data:
|
|
704
530
|
- SELECT id, description FROM "test"
|
|
705
531
|
`, { storageVersion }));
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
532
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
533
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
534
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
535
|
+
// Pre-setup
|
|
536
|
+
await writer.markAllSnapshotDone('1/1');
|
|
537
|
+
await writer.save({
|
|
538
|
+
sourceTable,
|
|
539
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
540
|
+
after: {
|
|
541
|
+
id: 'test1',
|
|
542
|
+
description: 'test1a'
|
|
543
|
+
},
|
|
544
|
+
afterReplicaId: test_utils.rid('test1')
|
|
545
|
+
});
|
|
546
|
+
await writer.save({
|
|
547
|
+
sourceTable,
|
|
548
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
549
|
+
after: {
|
|
550
|
+
id: 'test2',
|
|
551
|
+
description: 'test2a'
|
|
552
|
+
},
|
|
553
|
+
afterReplicaId: test_utils.rid('test2')
|
|
554
|
+
});
|
|
555
|
+
const result1 = await writer.flush();
|
|
556
|
+
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
557
|
+
// Test batch
|
|
558
|
+
// b
|
|
559
|
+
await writer.save({
|
|
560
|
+
sourceTable,
|
|
561
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
562
|
+
after: {
|
|
563
|
+
id: 'test1',
|
|
564
|
+
description: 'test1b'
|
|
565
|
+
},
|
|
566
|
+
afterReplicaId: test_utils.rid('test1')
|
|
567
|
+
});
|
|
568
|
+
await writer.save({
|
|
569
|
+
sourceTable,
|
|
570
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
571
|
+
before: {
|
|
572
|
+
id: 'test1'
|
|
573
|
+
},
|
|
574
|
+
beforeReplicaId: test_utils.rid('test1'),
|
|
575
|
+
after: {
|
|
576
|
+
id: 'test2',
|
|
577
|
+
description: 'test2b'
|
|
578
|
+
},
|
|
579
|
+
afterReplicaId: test_utils.rid('test2')
|
|
580
|
+
});
|
|
581
|
+
await writer.save({
|
|
582
|
+
sourceTable,
|
|
583
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
584
|
+
before: {
|
|
585
|
+
id: 'test2'
|
|
586
|
+
},
|
|
587
|
+
beforeReplicaId: test_utils.rid('test2'),
|
|
588
|
+
after: {
|
|
589
|
+
id: 'test3',
|
|
590
|
+
description: 'test3b'
|
|
591
|
+
},
|
|
592
|
+
afterReplicaId: test_utils.rid('test3')
|
|
593
|
+
});
|
|
594
|
+
// c
|
|
595
|
+
await writer.save({
|
|
596
|
+
sourceTable,
|
|
597
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
598
|
+
after: {
|
|
599
|
+
id: 'test2',
|
|
600
|
+
description: 'test2c'
|
|
601
|
+
},
|
|
602
|
+
afterReplicaId: test_utils.rid('test2')
|
|
603
|
+
});
|
|
604
|
+
// d
|
|
605
|
+
await writer.save({
|
|
606
|
+
sourceTable,
|
|
607
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
608
|
+
after: {
|
|
609
|
+
id: 'test4',
|
|
610
|
+
description: 'test4d'
|
|
611
|
+
},
|
|
612
|
+
afterReplicaId: test_utils.rid('test4')
|
|
613
|
+
});
|
|
614
|
+
await writer.save({
|
|
615
|
+
sourceTable,
|
|
616
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
617
|
+
before: {
|
|
618
|
+
id: 'test4'
|
|
619
|
+
},
|
|
620
|
+
beforeReplicaId: test_utils.rid('test4'),
|
|
621
|
+
after: {
|
|
622
|
+
id: 'test5',
|
|
623
|
+
description: 'test5d'
|
|
624
|
+
},
|
|
625
|
+
afterReplicaId: test_utils.rid('test5')
|
|
626
|
+
});
|
|
627
|
+
const result2 = await writer.flush();
|
|
628
|
+
const checkpoint2 = result2.flushed_op;
|
|
629
|
+
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
630
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
631
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
632
|
+
return {
|
|
633
|
+
op: d.op,
|
|
634
|
+
object_id: d.object_id,
|
|
635
|
+
data: normalizeOplogData(d.data)
|
|
636
|
+
};
|
|
637
|
+
});
|
|
638
|
+
// Operations must be in this order
|
|
639
|
+
expect(data).toEqual([
|
|
732
640
|
// b
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
description: 'test1b'
|
|
739
|
-
},
|
|
740
|
-
afterReplicaId: test_utils.rid('test1')
|
|
741
|
-
});
|
|
742
|
-
await writer.save({
|
|
743
|
-
sourceTable,
|
|
744
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
745
|
-
before: {
|
|
746
|
-
id: 'test1'
|
|
747
|
-
},
|
|
748
|
-
beforeReplicaId: test_utils.rid('test1'),
|
|
749
|
-
after: {
|
|
750
|
-
id: 'test2',
|
|
751
|
-
description: 'test2b'
|
|
752
|
-
},
|
|
753
|
-
afterReplicaId: test_utils.rid('test2')
|
|
754
|
-
});
|
|
755
|
-
await writer.save({
|
|
756
|
-
sourceTable,
|
|
757
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
758
|
-
before: {
|
|
759
|
-
id: 'test2'
|
|
760
|
-
},
|
|
761
|
-
beforeReplicaId: test_utils.rid('test2'),
|
|
762
|
-
after: {
|
|
763
|
-
id: 'test3',
|
|
764
|
-
description: 'test3b'
|
|
765
|
-
},
|
|
766
|
-
afterReplicaId: test_utils.rid('test3')
|
|
767
|
-
});
|
|
641
|
+
{ op: 'PUT', object_id: 'test1', data: JSON.stringify({ id: 'test1', description: 'test1b' }) },
|
|
642
|
+
{ op: 'REMOVE', object_id: 'test1', data: null },
|
|
643
|
+
{ op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2b' }) },
|
|
644
|
+
{ op: 'REMOVE', object_id: 'test2', data: null },
|
|
645
|
+
{ op: 'PUT', object_id: 'test3', data: JSON.stringify({ id: 'test3', description: 'test3b' }) },
|
|
768
646
|
// c
|
|
769
|
-
|
|
770
|
-
sourceTable,
|
|
771
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
772
|
-
after: {
|
|
773
|
-
id: 'test2',
|
|
774
|
-
description: 'test2c'
|
|
775
|
-
},
|
|
776
|
-
afterReplicaId: test_utils.rid('test2')
|
|
777
|
-
});
|
|
647
|
+
{ op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2c' }) },
|
|
778
648
|
// d
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
id: 'test4',
|
|
784
|
-
description: 'test4d'
|
|
785
|
-
},
|
|
786
|
-
afterReplicaId: test_utils.rid('test4')
|
|
787
|
-
});
|
|
788
|
-
await writer.save({
|
|
789
|
-
sourceTable,
|
|
790
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
791
|
-
before: {
|
|
792
|
-
id: 'test4'
|
|
793
|
-
},
|
|
794
|
-
beforeReplicaId: test_utils.rid('test4'),
|
|
795
|
-
after: {
|
|
796
|
-
id: 'test5',
|
|
797
|
-
description: 'test5d'
|
|
798
|
-
},
|
|
799
|
-
afterReplicaId: test_utils.rid('test5')
|
|
800
|
-
});
|
|
801
|
-
const result2 = await writer.flush();
|
|
802
|
-
const checkpoint2 = result2.flushed_op;
|
|
803
|
-
const request = bucketRequest(syncRules, 'global[]', checkpoint1);
|
|
804
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
805
|
-
const data = batch[0].chunkData.data.map((d) => {
|
|
806
|
-
return {
|
|
807
|
-
op: d.op,
|
|
808
|
-
object_id: d.object_id,
|
|
809
|
-
data: normalizeOplogData(d.data)
|
|
810
|
-
};
|
|
811
|
-
});
|
|
812
|
-
// Operations must be in this order
|
|
813
|
-
expect(data).toEqual([
|
|
814
|
-
// b
|
|
815
|
-
{ op: 'PUT', object_id: 'test1', data: JSON.stringify({ id: 'test1', description: 'test1b' }) },
|
|
816
|
-
{ op: 'REMOVE', object_id: 'test1', data: null },
|
|
817
|
-
{ op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2b' }) },
|
|
818
|
-
{ op: 'REMOVE', object_id: 'test2', data: null },
|
|
819
|
-
{ op: 'PUT', object_id: 'test3', data: JSON.stringify({ id: 'test3', description: 'test3b' }) },
|
|
820
|
-
// c
|
|
821
|
-
{ op: 'PUT', object_id: 'test2', data: JSON.stringify({ id: 'test2', description: 'test2c' }) },
|
|
822
|
-
// d
|
|
823
|
-
{ op: 'PUT', object_id: 'test4', data: JSON.stringify({ id: 'test4', description: 'test4d' }) },
|
|
824
|
-
{ op: 'REMOVE', object_id: 'test4', data: null },
|
|
825
|
-
{ op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
|
|
826
|
-
]);
|
|
827
|
-
}
|
|
828
|
-
catch (e_11) {
|
|
829
|
-
env_11.error = e_11;
|
|
830
|
-
env_11.hasError = true;
|
|
831
|
-
}
|
|
832
|
-
finally {
|
|
833
|
-
const result_11 = __disposeResources(env_11);
|
|
834
|
-
if (result_11)
|
|
835
|
-
await result_11;
|
|
836
|
-
}
|
|
649
|
+
{ op: 'PUT', object_id: 'test4', data: JSON.stringify({ id: 'test4', description: 'test4d' }) },
|
|
650
|
+
{ op: 'REMOVE', object_id: 'test4', data: null },
|
|
651
|
+
{ op: 'PUT', object_id: 'test5', data: JSON.stringify({ id: 'test5', description: 'test5d' }) }
|
|
652
|
+
]);
|
|
837
653
|
});
|
|
838
654
|
test('changed data with replica identity full', async () => {
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
845
|
-
]);
|
|
846
|
-
}
|
|
847
|
-
const factory = __addDisposableResource(env_12, await generateStorageFactory(), true);
|
|
848
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
849
|
-
bucket_definitions:
|
|
850
|
-
global:
|
|
851
|
-
data:
|
|
852
|
-
- SELECT id, description FROM "test"
|
|
853
|
-
`, {
|
|
854
|
-
storageVersion
|
|
855
|
-
}));
|
|
856
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
857
|
-
const writer = __addDisposableResource(env_12, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
858
|
-
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
859
|
-
// Pre-setup
|
|
860
|
-
await writer.markAllSnapshotDone('1/1');
|
|
861
|
-
await writer.save({
|
|
862
|
-
sourceTable,
|
|
863
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
864
|
-
after: {
|
|
865
|
-
id: 'test1',
|
|
866
|
-
description: 'test1a'
|
|
867
|
-
},
|
|
868
|
-
afterReplicaId: rid2('test1', 'test1a')
|
|
869
|
-
});
|
|
870
|
-
const result1 = await writer.flush();
|
|
871
|
-
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
872
|
-
// Unchanged, but has a before id
|
|
873
|
-
await writer.save({
|
|
874
|
-
sourceTable,
|
|
875
|
-
tag: storage.SaveOperationTag.UPDATE,
|
|
876
|
-
before: {
|
|
877
|
-
id: 'test1',
|
|
878
|
-
description: 'test1a'
|
|
879
|
-
},
|
|
880
|
-
beforeReplicaId: rid2('test1', 'test1a'),
|
|
881
|
-
after: {
|
|
882
|
-
id: 'test1',
|
|
883
|
-
description: 'test1b'
|
|
884
|
-
},
|
|
885
|
-
afterReplicaId: rid2('test1', 'test1b')
|
|
886
|
-
});
|
|
887
|
-
const result2 = await writer.flush();
|
|
888
|
-
// Delete
|
|
889
|
-
await writer.save({
|
|
890
|
-
sourceTable,
|
|
891
|
-
tag: storage.SaveOperationTag.DELETE,
|
|
892
|
-
before: {
|
|
893
|
-
id: 'test1',
|
|
894
|
-
description: 'test1b'
|
|
895
|
-
},
|
|
896
|
-
beforeReplicaId: rid2('test1', 'test1b'),
|
|
897
|
-
after: undefined
|
|
898
|
-
});
|
|
899
|
-
const result3 = await writer.flush();
|
|
900
|
-
const checkpoint3 = result3.flushed_op;
|
|
901
|
-
const request = bucketRequest(syncRules, 'global[]');
|
|
902
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
|
|
903
|
-
const data = batch[0].chunkData.data.map((d) => {
|
|
904
|
-
return {
|
|
905
|
-
op: d.op,
|
|
906
|
-
object_id: d.object_id,
|
|
907
|
-
data: normalizeOplogData(d.data),
|
|
908
|
-
subkey: d.subkey
|
|
909
|
-
};
|
|
910
|
-
});
|
|
911
|
-
// Operations must be in this order
|
|
912
|
-
expect(data).toEqual([
|
|
913
|
-
// 2
|
|
914
|
-
// The REMOVE is expected because the subkey changes
|
|
915
|
-
{
|
|
916
|
-
op: 'REMOVE',
|
|
917
|
-
object_id: 'test1',
|
|
918
|
-
data: null,
|
|
919
|
-
subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
|
|
920
|
-
},
|
|
921
|
-
{
|
|
922
|
-
op: 'PUT',
|
|
923
|
-
object_id: 'test1',
|
|
924
|
-
data: JSON.stringify({ id: 'test1', description: 'test1b' }),
|
|
925
|
-
subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
|
|
926
|
-
},
|
|
927
|
-
// 3
|
|
928
|
-
{
|
|
929
|
-
op: 'REMOVE',
|
|
930
|
-
object_id: 'test1',
|
|
931
|
-
data: null,
|
|
932
|
-
subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
|
|
933
|
-
}
|
|
934
|
-
]);
|
|
935
|
-
}
|
|
936
|
-
catch (e_12) {
|
|
937
|
-
env_12.error = e_12;
|
|
938
|
-
env_12.hasError = true;
|
|
939
|
-
}
|
|
940
|
-
finally {
|
|
941
|
-
const result_12 = __disposeResources(env_12);
|
|
942
|
-
if (result_12)
|
|
943
|
-
await result_12;
|
|
655
|
+
function rid2(id, description) {
|
|
656
|
+
return getUuidReplicaIdentityBson({ id, description }, [
|
|
657
|
+
{ name: 'id', type: 'VARCHAR', typeId: 25 },
|
|
658
|
+
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
659
|
+
]);
|
|
944
660
|
}
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
const env_13 = { stack: [], error: void 0, hasError: false };
|
|
948
|
-
try {
|
|
949
|
-
function rid2(id, description) {
|
|
950
|
-
return getUuidReplicaIdentityBson({ id, description }, [
|
|
951
|
-
{ name: 'id', type: 'VARCHAR', typeId: 25 },
|
|
952
|
-
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
953
|
-
]);
|
|
954
|
-
}
|
|
955
|
-
const factory = __addDisposableResource(env_13, await generateStorageFactory(), true);
|
|
956
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
661
|
+
await using factory = await generateStorageFactory();
|
|
662
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
957
663
|
bucket_definitions:
|
|
958
664
|
global:
|
|
959
665
|
data:
|
|
960
666
|
- SELECT id, description FROM "test"
|
|
961
667
|
`, {
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
668
|
+
storageVersion
|
|
669
|
+
}));
|
|
670
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
671
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
672
|
+
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
673
|
+
// Pre-setup
|
|
674
|
+
await writer.markAllSnapshotDone('1/1');
|
|
675
|
+
await writer.save({
|
|
676
|
+
sourceTable,
|
|
677
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
678
|
+
after: {
|
|
679
|
+
id: 'test1',
|
|
680
|
+
description: 'test1a'
|
|
681
|
+
},
|
|
682
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
683
|
+
});
|
|
684
|
+
const result1 = await writer.flush();
|
|
685
|
+
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
686
|
+
// Unchanged, but has a before id
|
|
687
|
+
await writer.save({
|
|
688
|
+
sourceTable,
|
|
689
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
690
|
+
before: {
|
|
691
|
+
id: 'test1',
|
|
692
|
+
description: 'test1a'
|
|
693
|
+
},
|
|
694
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
695
|
+
after: {
|
|
696
|
+
id: 'test1',
|
|
697
|
+
description: 'test1b'
|
|
698
|
+
},
|
|
699
|
+
afterReplicaId: rid2('test1', 'test1b')
|
|
700
|
+
});
|
|
701
|
+
const result2 = await writer.flush();
|
|
702
|
+
// Delete
|
|
703
|
+
await writer.save({
|
|
704
|
+
sourceTable,
|
|
705
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
706
|
+
before: {
|
|
707
|
+
id: 'test1',
|
|
708
|
+
description: 'test1b'
|
|
709
|
+
},
|
|
710
|
+
beforeReplicaId: rid2('test1', 'test1b'),
|
|
711
|
+
after: undefined
|
|
712
|
+
});
|
|
713
|
+
const result3 = await writer.flush();
|
|
714
|
+
const checkpoint3 = result3.flushed_op;
|
|
715
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
716
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
|
|
717
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
718
|
+
return {
|
|
719
|
+
op: d.op,
|
|
720
|
+
object_id: d.object_id,
|
|
721
|
+
data: normalizeOplogData(d.data),
|
|
722
|
+
subkey: d.subkey
|
|
723
|
+
};
|
|
724
|
+
});
|
|
725
|
+
// Operations must be in this order
|
|
726
|
+
expect(data).toEqual([
|
|
727
|
+
// 2
|
|
728
|
+
// The REMOVE is expected because the subkey changes
|
|
729
|
+
{
|
|
730
|
+
op: 'REMOVE',
|
|
731
|
+
object_id: 'test1',
|
|
732
|
+
data: null,
|
|
733
|
+
subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
|
|
734
|
+
},
|
|
735
|
+
{
|
|
736
|
+
op: 'PUT',
|
|
737
|
+
object_id: 'test1',
|
|
738
|
+
data: JSON.stringify({ id: 'test1', description: 'test1b' }),
|
|
739
|
+
subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
|
|
740
|
+
},
|
|
741
|
+
// 3
|
|
742
|
+
{
|
|
743
|
+
op: 'REMOVE',
|
|
744
|
+
object_id: 'test1',
|
|
745
|
+
data: null,
|
|
746
|
+
subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
|
|
747
|
+
}
|
|
748
|
+
]);
|
|
749
|
+
});
|
|
750
|
+
test('unchanged data with replica identity full', async () => {
|
|
751
|
+
function rid2(id, description) {
|
|
752
|
+
return getUuidReplicaIdentityBson({ id, description }, [
|
|
753
|
+
{ name: 'id', type: 'VARCHAR', typeId: 25 },
|
|
754
|
+
{ name: 'description', type: 'VARCHAR', typeId: 25 }
|
|
1035
755
|
]);
|
|
1036
756
|
}
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
}
|
|
757
|
+
await using factory = await generateStorageFactory();
|
|
758
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
759
|
+
bucket_definitions:
|
|
760
|
+
global:
|
|
761
|
+
data:
|
|
762
|
+
- SELECT id, description FROM "test"
|
|
763
|
+
`, {
|
|
764
|
+
storageVersion
|
|
765
|
+
}));
|
|
766
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
767
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
768
|
+
const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
|
|
769
|
+
// Pre-setup
|
|
770
|
+
await writer.markAllSnapshotDone('1/1');
|
|
771
|
+
await writer.save({
|
|
772
|
+
sourceTable,
|
|
773
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
774
|
+
after: {
|
|
775
|
+
id: 'test1',
|
|
776
|
+
description: 'test1a'
|
|
777
|
+
},
|
|
778
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
779
|
+
});
|
|
780
|
+
const result1 = await writer.flush();
|
|
781
|
+
const checkpoint1 = result1?.flushed_op ?? 0n;
|
|
782
|
+
// Unchanged, but has a before id
|
|
783
|
+
await writer.save({
|
|
784
|
+
sourceTable,
|
|
785
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
786
|
+
before: {
|
|
787
|
+
id: 'test1',
|
|
788
|
+
description: 'test1a'
|
|
789
|
+
},
|
|
790
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
791
|
+
after: {
|
|
792
|
+
id: 'test1',
|
|
793
|
+
description: 'test1a'
|
|
794
|
+
},
|
|
795
|
+
afterReplicaId: rid2('test1', 'test1a')
|
|
796
|
+
});
|
|
797
|
+
const result2 = await writer.flush();
|
|
798
|
+
// Delete
|
|
799
|
+
await writer.save({
|
|
800
|
+
sourceTable,
|
|
801
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
802
|
+
before: {
|
|
803
|
+
id: 'test1',
|
|
804
|
+
description: 'test1a'
|
|
805
|
+
},
|
|
806
|
+
beforeReplicaId: rid2('test1', 'test1a'),
|
|
807
|
+
after: undefined
|
|
808
|
+
});
|
|
809
|
+
const result3 = await writer.flush();
|
|
810
|
+
const checkpoint3 = result3.flushed_op;
|
|
811
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
812
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]));
|
|
813
|
+
const data = batch[0].chunkData.data.map((d) => {
|
|
814
|
+
return {
|
|
815
|
+
op: d.op,
|
|
816
|
+
object_id: d.object_id,
|
|
817
|
+
data: normalizeOplogData(d.data),
|
|
818
|
+
subkey: d.subkey
|
|
819
|
+
};
|
|
820
|
+
});
|
|
821
|
+
// Operations must be in this order
|
|
822
|
+
expect(data).toEqual([
|
|
823
|
+
// 2
|
|
824
|
+
{
|
|
825
|
+
op: 'PUT',
|
|
826
|
+
object_id: 'test1',
|
|
827
|
+
data: JSON.stringify({ id: 'test1', description: 'test1a' }),
|
|
828
|
+
subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
|
|
829
|
+
},
|
|
830
|
+
// 3
|
|
831
|
+
{
|
|
832
|
+
op: 'REMOVE',
|
|
833
|
+
object_id: 'test1',
|
|
834
|
+
data: null,
|
|
835
|
+
subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
|
|
836
|
+
}
|
|
837
|
+
]);
|
|
1046
838
|
});
|
|
1047
839
|
test('large batch', async () => {
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
const factory = __addDisposableResource(env_14, await generateStorageFactory(), true);
|
|
1055
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
840
|
+
// Test syncing a batch of data that is small in count,
|
|
841
|
+
// but large enough in size to be split over multiple returned batches.
|
|
842
|
+
// The specific batch splits is an implementation detail of the storage driver,
|
|
843
|
+
// and the test will have to updated when other implementations are added.
|
|
844
|
+
await using factory = await generateStorageFactory();
|
|
845
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1056
846
|
bucket_definitions:
|
|
1057
847
|
global:
|
|
1058
848
|
data:
|
|
1059
849
|
- SELECT id, description FROM "%"
|
|
1060
850
|
`, {
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
}
|
|
1135
|
-
catch (e_14) {
|
|
1136
|
-
env_14.error = e_14;
|
|
1137
|
-
env_14.hasError = true;
|
|
1138
|
-
}
|
|
1139
|
-
finally {
|
|
1140
|
-
const result_14 = __disposeResources(env_14);
|
|
1141
|
-
if (result_14)
|
|
1142
|
-
await result_14;
|
|
1143
|
-
}
|
|
851
|
+
storageVersion
|
|
852
|
+
}));
|
|
853
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
854
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
855
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
856
|
+
await writer.markAllSnapshotDone('1/1');
|
|
857
|
+
const largeDescription = '0123456789'.repeat(12_000_00);
|
|
858
|
+
await writer.save({
|
|
859
|
+
sourceTable,
|
|
860
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
861
|
+
after: {
|
|
862
|
+
id: 'test1',
|
|
863
|
+
description: 'test1'
|
|
864
|
+
},
|
|
865
|
+
afterReplicaId: test_utils.rid('test1')
|
|
866
|
+
});
|
|
867
|
+
await writer.save({
|
|
868
|
+
sourceTable,
|
|
869
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
870
|
+
after: {
|
|
871
|
+
id: 'large1',
|
|
872
|
+
description: largeDescription
|
|
873
|
+
},
|
|
874
|
+
afterReplicaId: test_utils.rid('large1')
|
|
875
|
+
});
|
|
876
|
+
// Large enough to split the returned batch
|
|
877
|
+
await writer.save({
|
|
878
|
+
sourceTable,
|
|
879
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
880
|
+
after: {
|
|
881
|
+
id: 'large2',
|
|
882
|
+
description: largeDescription
|
|
883
|
+
},
|
|
884
|
+
afterReplicaId: test_utils.rid('large2')
|
|
885
|
+
});
|
|
886
|
+
await writer.save({
|
|
887
|
+
sourceTable,
|
|
888
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
889
|
+
after: {
|
|
890
|
+
id: 'test3',
|
|
891
|
+
description: 'test3'
|
|
892
|
+
},
|
|
893
|
+
afterReplicaId: test_utils.rid('test3')
|
|
894
|
+
});
|
|
895
|
+
await writer.commit('1/1');
|
|
896
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
897
|
+
const options = {
|
|
898
|
+
chunkLimitBytes: 16 * 1024 * 1024
|
|
899
|
+
};
|
|
900
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
901
|
+
const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options));
|
|
902
|
+
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
903
|
+
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
904
|
+
{ op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 }
|
|
905
|
+
]);
|
|
906
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
907
|
+
after: '0',
|
|
908
|
+
has_more: true,
|
|
909
|
+
next_after: '2'
|
|
910
|
+
});
|
|
911
|
+
const batch2 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options));
|
|
912
|
+
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
913
|
+
{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
|
|
914
|
+
{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
|
|
915
|
+
]);
|
|
916
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual({
|
|
917
|
+
after: '2',
|
|
918
|
+
has_more: false,
|
|
919
|
+
next_after: '4'
|
|
920
|
+
});
|
|
921
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options));
|
|
922
|
+
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
923
|
+
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
1144
924
|
});
|
|
1145
925
|
test('long batch', async () => {
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
const factory = __addDisposableResource(env_15, await generateStorageFactory(), true);
|
|
1150
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
926
|
+
// Test syncing a batch of data that is limited by count.
|
|
927
|
+
await using factory = await generateStorageFactory();
|
|
928
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1151
929
|
bucket_definitions:
|
|
1152
930
|
global:
|
|
1153
931
|
data:
|
|
1154
932
|
- SELECT id, description FROM "%"
|
|
1155
933
|
`, {
|
|
1156
|
-
|
|
1157
|
-
|
|
934
|
+
storageVersion
|
|
935
|
+
}));
|
|
936
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
937
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
938
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
939
|
+
await writer.markAllSnapshotDone('1/1');
|
|
940
|
+
for (let i = 1; i <= 6; i++) {
|
|
941
|
+
await writer.save({
|
|
942
|
+
sourceTable,
|
|
943
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
944
|
+
after: {
|
|
945
|
+
id: `test${i}`,
|
|
946
|
+
description: `test${i}`
|
|
947
|
+
},
|
|
948
|
+
afterReplicaId: `test${i}`
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
await writer.commit('1/1');
|
|
952
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
953
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
954
|
+
const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 }));
|
|
955
|
+
expect(test_utils.getBatchData(batch1)).toEqual([
|
|
956
|
+
{ op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
|
|
957
|
+
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
958
|
+
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
959
|
+
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
|
|
960
|
+
]);
|
|
961
|
+
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
962
|
+
after: '0',
|
|
963
|
+
has_more: true,
|
|
964
|
+
next_after: '4'
|
|
965
|
+
});
|
|
966
|
+
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
967
|
+
limit: 4
|
|
968
|
+
}));
|
|
969
|
+
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
970
|
+
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
971
|
+
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
|
|
972
|
+
]);
|
|
973
|
+
expect(test_utils.getBatchMeta(batch2)).toEqual({
|
|
974
|
+
after: '4',
|
|
975
|
+
has_more: false,
|
|
976
|
+
next_after: '6'
|
|
977
|
+
});
|
|
978
|
+
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
979
|
+
limit: 4
|
|
980
|
+
}));
|
|
981
|
+
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
982
|
+
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
983
|
+
});
|
|
984
|
+
describe('batch has_more', () => {
|
|
985
|
+
const setup = async (options) => {
|
|
986
|
+
await using factory = await generateStorageFactory();
|
|
987
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
988
|
+
bucket_definitions:
|
|
989
|
+
global1:
|
|
990
|
+
data:
|
|
991
|
+
- SELECT id, description FROM test WHERE bucket = 'global1'
|
|
992
|
+
global2:
|
|
993
|
+
data:
|
|
994
|
+
- SELECT id, description FROM test WHERE bucket = 'global2'
|
|
995
|
+
`, { storageVersion }));
|
|
1158
996
|
const bucketStorage = factory.getInstance(syncRules);
|
|
1159
|
-
|
|
997
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1160
998
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1161
999
|
await writer.markAllSnapshotDone('1/1');
|
|
1162
|
-
for (let i = 1; i <=
|
|
1000
|
+
for (let i = 1; i <= 10; i++) {
|
|
1163
1001
|
await writer.save({
|
|
1164
1002
|
sourceTable,
|
|
1165
1003
|
tag: storage.SaveOperationTag.INSERT,
|
|
1166
1004
|
after: {
|
|
1167
1005
|
id: `test${i}`,
|
|
1168
|
-
description: `test${i}
|
|
1006
|
+
description: `test${i}`,
|
|
1007
|
+
bucket: i == 1 ? 'global1' : 'global2'
|
|
1169
1008
|
},
|
|
1170
1009
|
afterReplicaId: `test${i}`
|
|
1171
1010
|
});
|
|
1172
1011
|
}
|
|
1173
1012
|
await writer.commit('1/1');
|
|
1174
1013
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1175
|
-
const
|
|
1176
|
-
const
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
{ op_id: '2', op: 'PUT', object_id: 'test2', checksum: 730027011 },
|
|
1180
|
-
{ op_id: '3', op: 'PUT', object_id: 'test3', checksum: 1359888332 },
|
|
1181
|
-
{ op_id: '4', op: 'PUT', object_id: 'test4', checksum: 2049153252 }
|
|
1182
|
-
]);
|
|
1183
|
-
expect(test_utils.getBatchMeta(batch1)).toEqual({
|
|
1184
|
-
after: '0',
|
|
1185
|
-
has_more: true,
|
|
1186
|
-
next_after: '4'
|
|
1187
|
-
});
|
|
1188
|
-
const batch2 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], {
|
|
1189
|
-
limit: 4
|
|
1190
|
-
}));
|
|
1191
|
-
expect(test_utils.getBatchData(batch2)).toEqual([
|
|
1192
|
-
{ op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 },
|
|
1193
|
-
{ op_id: '6', op: 'PUT', object_id: 'test6', checksum: 1974820016 }
|
|
1194
|
-
]);
|
|
1195
|
-
expect(test_utils.getBatchMeta(batch2)).toEqual({
|
|
1196
|
-
after: '4',
|
|
1197
|
-
has_more: false,
|
|
1198
|
-
next_after: '6'
|
|
1199
|
-
});
|
|
1200
|
-
const batch3 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], {
|
|
1201
|
-
limit: 4
|
|
1202
|
-
}));
|
|
1203
|
-
expect(test_utils.getBatchData(batch3)).toEqual([]);
|
|
1204
|
-
expect(test_utils.getBatchMeta(batch3)).toEqual(null);
|
|
1205
|
-
}
|
|
1206
|
-
catch (e_15) {
|
|
1207
|
-
env_15.error = e_15;
|
|
1208
|
-
env_15.hasError = true;
|
|
1209
|
-
}
|
|
1210
|
-
finally {
|
|
1211
|
-
const result_15 = __disposeResources(env_15);
|
|
1212
|
-
if (result_15)
|
|
1213
|
-
await result_15;
|
|
1214
|
-
}
|
|
1215
|
-
});
|
|
1216
|
-
describe('batch has_more', () => {
|
|
1217
|
-
const setup = async (options) => {
|
|
1218
|
-
const env_16 = { stack: [], error: void 0, hasError: false };
|
|
1219
|
-
try {
|
|
1220
|
-
const factory = __addDisposableResource(env_16, await generateStorageFactory(), true);
|
|
1221
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1222
|
-
bucket_definitions:
|
|
1223
|
-
global1:
|
|
1224
|
-
data:
|
|
1225
|
-
- SELECT id, description FROM test WHERE bucket = 'global1'
|
|
1226
|
-
global2:
|
|
1227
|
-
data:
|
|
1228
|
-
- SELECT id, description FROM test WHERE bucket = 'global2'
|
|
1229
|
-
`, { storageVersion }));
|
|
1230
|
-
const bucketStorage = factory.getInstance(syncRules);
|
|
1231
|
-
const writer = __addDisposableResource(env_16, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
|
|
1232
|
-
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1233
|
-
await writer.markAllSnapshotDone('1/1');
|
|
1234
|
-
for (let i = 1; i <= 10; i++) {
|
|
1235
|
-
await writer.save({
|
|
1236
|
-
sourceTable,
|
|
1237
|
-
tag: storage.SaveOperationTag.INSERT,
|
|
1238
|
-
after: {
|
|
1239
|
-
id: `test${i}`,
|
|
1240
|
-
description: `test${i}`,
|
|
1241
|
-
bucket: i == 1 ? 'global1' : 'global2'
|
|
1242
|
-
},
|
|
1243
|
-
afterReplicaId: `test${i}`
|
|
1244
|
-
});
|
|
1245
|
-
}
|
|
1246
|
-
await writer.commit('1/1');
|
|
1247
|
-
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1248
|
-
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1249
|
-
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1250
|
-
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
|
|
1251
|
-
return { batch, global1Request, global2Request };
|
|
1252
|
-
}
|
|
1253
|
-
catch (e_16) {
|
|
1254
|
-
env_16.error = e_16;
|
|
1255
|
-
env_16.hasError = true;
|
|
1256
|
-
}
|
|
1257
|
-
finally {
|
|
1258
|
-
const result_16 = __disposeResources(env_16);
|
|
1259
|
-
if (result_16)
|
|
1260
|
-
await result_16;
|
|
1261
|
-
}
|
|
1014
|
+
const global1Request = bucketRequest(syncRules, 'global1[]', 0n);
|
|
1015
|
+
const global2Request = bucketRequest(syncRules, 'global2[]', 0n);
|
|
1016
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options));
|
|
1017
|
+
return { batch, global1Request, global2Request };
|
|
1262
1018
|
};
|
|
1263
1019
|
test('batch has_more (1)', async () => {
|
|
1264
1020
|
const { batch, global1Request, global2Request } = await setup({ limit: 5 });
|
|
@@ -1349,330 +1105,247 @@ bucket_definitions:
|
|
|
1349
1105
|
});
|
|
1350
1106
|
});
|
|
1351
1107
|
test('empty storage metrics', async () => {
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
await f.getStorageMetrics();
|
|
1367
|
-
}
|
|
1368
|
-
catch (e_17) {
|
|
1369
|
-
env_17.error = e_17;
|
|
1370
|
-
env_17.hasError = true;
|
|
1371
|
-
}
|
|
1372
|
-
finally {
|
|
1373
|
-
const result_17 = __disposeResources(env_17);
|
|
1374
|
-
if (result_17)
|
|
1375
|
-
await result_17;
|
|
1376
|
-
}
|
|
1108
|
+
await using f = await generateStorageFactory({ dropAll: true });
|
|
1109
|
+
const metrics = await f.getStorageMetrics();
|
|
1110
|
+
expect(metrics).toEqual({
|
|
1111
|
+
operations_size_bytes: 0,
|
|
1112
|
+
parameters_size_bytes: 0,
|
|
1113
|
+
replication_size_bytes: 0
|
|
1114
|
+
});
|
|
1115
|
+
const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}'));
|
|
1116
|
+
const storage = f.getInstance(r.persisted_sync_rules);
|
|
1117
|
+
await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1118
|
+
await writer.markAllSnapshotDone('1/0');
|
|
1119
|
+
await writer.keepalive('1/0');
|
|
1120
|
+
await f.getStorageMetrics();
|
|
1121
|
+
// We don't care about the specific values here
|
|
1377
1122
|
});
|
|
1378
1123
|
test('op_id initialization edge case', async () => {
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
const factory = __addDisposableResource(env_18, await generateStorageFactory(), true);
|
|
1385
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1124
|
+
// Test syncing a batch of data that is small in count,
|
|
1125
|
+
// but large enough in size to be split over multiple returned chunks.
|
|
1126
|
+
// Similar to the above test, but splits over 1MB chunks.
|
|
1127
|
+
await using factory = await generateStorageFactory();
|
|
1128
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1386
1129
|
bucket_definitions:
|
|
1387
1130
|
global:
|
|
1388
1131
|
data:
|
|
1389
1132
|
- SELECT id FROM test
|
|
1390
1133
|
- SELECT id FROM test_ignore WHERE false
|
|
1391
1134
|
`, {
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
}
|
|
1424
|
-
catch (e_18) {
|
|
1425
|
-
env_18.error = e_18;
|
|
1426
|
-
env_18.hasError = true;
|
|
1427
|
-
}
|
|
1428
|
-
finally {
|
|
1429
|
-
const result_18 = __disposeResources(env_18);
|
|
1430
|
-
if (result_18)
|
|
1431
|
-
await result_18;
|
|
1432
|
-
}
|
|
1135
|
+
storageVersion
|
|
1136
|
+
}));
|
|
1137
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1138
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1139
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1);
|
|
1140
|
+
const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2);
|
|
1141
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1142
|
+
// This saves a record to current_data, but not bucket_data.
|
|
1143
|
+
// This causes a checkpoint to be created without increasing the op_id sequence.
|
|
1144
|
+
await writer.save({
|
|
1145
|
+
sourceTable: sourceTableIgnore,
|
|
1146
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1147
|
+
after: {
|
|
1148
|
+
id: 'test1'
|
|
1149
|
+
},
|
|
1150
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1151
|
+
});
|
|
1152
|
+
const result1 = await writer.flush();
|
|
1153
|
+
const checkpoint1 = result1.flushed_op;
|
|
1154
|
+
await writer.save({
|
|
1155
|
+
sourceTable: sourceTable,
|
|
1156
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1157
|
+
after: {
|
|
1158
|
+
id: 'test2'
|
|
1159
|
+
},
|
|
1160
|
+
afterReplicaId: test_utils.rid('test2')
|
|
1161
|
+
});
|
|
1162
|
+
const result2 = await writer.flush();
|
|
1163
|
+
const checkpoint2 = result2.flushed_op;
|
|
1164
|
+
// we expect 0n and 1n, or 1n and 2n.
|
|
1165
|
+
expect(checkpoint2).toBeGreaterThan(checkpoint1);
|
|
1433
1166
|
});
|
|
1434
1167
|
test('unchanged checksums', async () => {
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
const factory = __addDisposableResource(env_19, await generateStorageFactory(), true);
|
|
1438
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1168
|
+
await using factory = await generateStorageFactory();
|
|
1169
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1439
1170
|
bucket_definitions:
|
|
1440
1171
|
global:
|
|
1441
1172
|
data:
|
|
1442
1173
|
- SELECT client_id as id, description FROM "%"
|
|
1443
1174
|
`, {
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
}
|
|
1467
|
-
catch (e_19) {
|
|
1468
|
-
env_19.error = e_19;
|
|
1469
|
-
env_19.hasError = true;
|
|
1470
|
-
}
|
|
1471
|
-
finally {
|
|
1472
|
-
const result_19 = __disposeResources(env_19);
|
|
1473
|
-
if (result_19)
|
|
1474
|
-
await result_19;
|
|
1475
|
-
}
|
|
1175
|
+
storageVersion
|
|
1176
|
+
}));
|
|
1177
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1178
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1179
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1180
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1181
|
+
await writer.save({
|
|
1182
|
+
sourceTable,
|
|
1183
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1184
|
+
after: {
|
|
1185
|
+
id: 'test1',
|
|
1186
|
+
description: 'test1a'
|
|
1187
|
+
},
|
|
1188
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1189
|
+
});
|
|
1190
|
+
await writer.commit('1/1');
|
|
1191
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1192
|
+
const request = bucketRequest(syncRules, 'global[]');
|
|
1193
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()];
|
|
1194
|
+
expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1195
|
+
const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()];
|
|
1196
|
+
expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]);
|
|
1476
1197
|
});
|
|
1477
1198
|
testChecksumBatching(config);
|
|
1478
1199
|
test('empty checkpoints (1)', async () => {
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
const factory = __addDisposableResource(env_20, await generateStorageFactory(), true);
|
|
1482
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1200
|
+
await using factory = await generateStorageFactory();
|
|
1201
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1483
1202
|
bucket_definitions:
|
|
1484
1203
|
global:
|
|
1485
1204
|
data:
|
|
1486
1205
|
- SELECT id, description FROM "%"
|
|
1487
1206
|
`, { storageVersion }));
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
}
|
|
1506
|
-
catch (e_20) {
|
|
1507
|
-
env_20.error = e_20;
|
|
1508
|
-
env_20.hasError = true;
|
|
1509
|
-
}
|
|
1510
|
-
finally {
|
|
1511
|
-
const result_20 = __disposeResources(env_20);
|
|
1512
|
-
if (result_20)
|
|
1513
|
-
await result_20;
|
|
1514
|
-
}
|
|
1207
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1208
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1209
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1210
|
+
await writer.commit('1/1');
|
|
1211
|
+
const cp1 = await bucketStorage.getCheckpoint();
|
|
1212
|
+
expect(cp1.lsn).toEqual('1/1');
|
|
1213
|
+
await writer.commit('2/1', { createEmptyCheckpoints: true });
|
|
1214
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1215
|
+
expect(cp2.lsn).toEqual('2/1');
|
|
1216
|
+
await writer.keepalive('3/1');
|
|
1217
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1218
|
+
expect(cp3.lsn).toEqual('3/1');
|
|
1219
|
+
// For the last one, we skip creating empty checkpoints
|
|
1220
|
+
// This means the LSN stays at 3/1.
|
|
1221
|
+
await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1222
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1223
|
+
expect(cp4.lsn).toEqual('3/1');
|
|
1515
1224
|
});
|
|
1516
1225
|
test('empty checkpoints (2)', async () => {
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
const factory = __addDisposableResource(env_21, await generateStorageFactory(), true);
|
|
1520
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1226
|
+
await using factory = await generateStorageFactory();
|
|
1227
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1521
1228
|
bucket_definitions:
|
|
1522
1229
|
global:
|
|
1523
1230
|
data:
|
|
1524
1231
|
- SELECT id, description FROM "%"
|
|
1525
1232
|
`, {
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
}
|
|
1557
|
-
catch (e_21) {
|
|
1558
|
-
env_21.error = e_21;
|
|
1559
|
-
env_21.hasError = true;
|
|
1560
|
-
}
|
|
1561
|
-
finally {
|
|
1562
|
-
const result_21 = __disposeResources(env_21);
|
|
1563
|
-
if (result_21)
|
|
1564
|
-
await result_21;
|
|
1565
|
-
}
|
|
1233
|
+
storageVersion
|
|
1234
|
+
}));
|
|
1235
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1236
|
+
await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1237
|
+
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1238
|
+
const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
1239
|
+
// We simulate two concurrent batches, but sequential calls are enough for this test.
|
|
1240
|
+
await writer1.markAllSnapshotDone('1/1');
|
|
1241
|
+
await writer1.commit('1/1');
|
|
1242
|
+
await writer1.commit('2/1', { createEmptyCheckpoints: false });
|
|
1243
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1244
|
+
expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped
|
|
1245
|
+
await writer2.save({
|
|
1246
|
+
sourceTable,
|
|
1247
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1248
|
+
after: {
|
|
1249
|
+
id: 'test1',
|
|
1250
|
+
description: 'test1a'
|
|
1251
|
+
},
|
|
1252
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1253
|
+
});
|
|
1254
|
+
// This simulates what happens on a snapshot processor.
|
|
1255
|
+
// This may later change to a flush() rather than commit().
|
|
1256
|
+
await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN);
|
|
1257
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1258
|
+
expect(cp3.lsn).toEqual('1/1'); // Still unchanged
|
|
1259
|
+
// This now needs to advance the LSN, despite {createEmptyCheckpoints: false}
|
|
1260
|
+
await writer1.commit('4/1', { createEmptyCheckpoints: false });
|
|
1261
|
+
const cp4 = await bucketStorage.getCheckpoint();
|
|
1262
|
+
expect(cp4.lsn).toEqual('4/1');
|
|
1566
1263
|
});
|
|
1567
1264
|
test('empty checkpoints (sync rule activation)', async () => {
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
const factory = __addDisposableResource(env_22, await generateStorageFactory(), true);
|
|
1571
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1265
|
+
await using factory = await generateStorageFactory();
|
|
1266
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1572
1267
|
bucket_definitions:
|
|
1573
1268
|
global:
|
|
1574
1269
|
data:
|
|
1575
1270
|
- SELECT id, description FROM "%"
|
|
1576
1271
|
`, {
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
}
|
|
1607
|
-
catch (e_22) {
|
|
1608
|
-
env_22.error = e_22;
|
|
1609
|
-
env_22.hasError = true;
|
|
1610
|
-
}
|
|
1611
|
-
finally {
|
|
1612
|
-
const result_22 = __disposeResources(env_22);
|
|
1613
|
-
if (result_22)
|
|
1614
|
-
await result_22;
|
|
1615
|
-
}
|
|
1272
|
+
storageVersion
|
|
1273
|
+
}));
|
|
1274
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1275
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1276
|
+
const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false });
|
|
1277
|
+
expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1278
|
+
// Snapshot is only valid once we reach 3/1
|
|
1279
|
+
await writer.markAllSnapshotDone('3/1');
|
|
1280
|
+
// 2/1 < 3/1 - snapshot not valid yet, block checkpoint
|
|
1281
|
+
const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false });
|
|
1282
|
+
expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false });
|
|
1283
|
+
// No empty checkpoint should be created by the commit above.
|
|
1284
|
+
const cp1 = await bucketStorage.getCheckpoint();
|
|
1285
|
+
expect(cp1.lsn).toEqual(null);
|
|
1286
|
+
// After this commit, the snapshot should be valid.
|
|
1287
|
+
// We specifically check that this is done even if createEmptyCheckpoints: false.
|
|
1288
|
+
const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false });
|
|
1289
|
+
expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true });
|
|
1290
|
+
// Now, the checkpoint should advance the sync rules active.
|
|
1291
|
+
const cp2 = await bucketStorage.getCheckpoint();
|
|
1292
|
+
expect(cp2.lsn).toEqual('3/1');
|
|
1293
|
+
const activeSyncRules = await factory.getActiveSyncRulesContent();
|
|
1294
|
+
expect(activeSyncRules?.id).toEqual(syncRules.id);
|
|
1295
|
+
// At this point, it should be a truely empty checkpoint
|
|
1296
|
+
const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false });
|
|
1297
|
+
expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false });
|
|
1298
|
+
// Unchanged
|
|
1299
|
+
const cp3 = await bucketStorage.getCheckpoint();
|
|
1300
|
+
expect(cp3.lsn).toEqual('3/1');
|
|
1616
1301
|
});
|
|
1617
1302
|
test.runIf(storageVersion >= 3)('deleting while streaming', async () => {
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
const factory = __addDisposableResource(env_23, await generateStorageFactory(), true);
|
|
1621
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1303
|
+
await using factory = await generateStorageFactory();
|
|
1304
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1622
1305
|
bucket_definitions:
|
|
1623
1306
|
global:
|
|
1624
1307
|
data:
|
|
1625
1308
|
- SELECT id, description FROM "%"
|
|
1626
1309
|
`, {
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
}
|
|
1667
|
-
catch (e_23) {
|
|
1668
|
-
env_23.error = e_23;
|
|
1669
|
-
env_23.hasError = true;
|
|
1670
|
-
}
|
|
1671
|
-
finally {
|
|
1672
|
-
const result_23 = __disposeResources(env_23);
|
|
1673
|
-
if (result_23)
|
|
1674
|
-
await result_23;
|
|
1675
|
-
}
|
|
1310
|
+
storageVersion
|
|
1311
|
+
}));
|
|
1312
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1313
|
+
await using snapshotWriter = await bucketStorage.createWriter({
|
|
1314
|
+
...test_utils.BATCH_OPTIONS,
|
|
1315
|
+
skipExistingRows: true
|
|
1316
|
+
});
|
|
1317
|
+
await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1318
|
+
const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1);
|
|
1319
|
+
const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1);
|
|
1320
|
+
// We simulate two concurrent batches; separate writers are enough for this test.
|
|
1321
|
+
// For this test, we assume that we start with a row "test1", which is picked up by a snapshot
|
|
1322
|
+
// query, right before the delete is streamed. But the snapshot query is only persisted _after_
|
|
1323
|
+
// the delete is streamed, and we need to ensure that the streamed delete takes precedence.
|
|
1324
|
+
await streamingWriter.save({
|
|
1325
|
+
sourceTable: streamingTable,
|
|
1326
|
+
tag: storage.SaveOperationTag.DELETE,
|
|
1327
|
+
before: {
|
|
1328
|
+
id: 'test1'
|
|
1329
|
+
},
|
|
1330
|
+
beforeReplicaId: test_utils.rid('test1')
|
|
1331
|
+
});
|
|
1332
|
+
await streamingWriter.commit('2/1');
|
|
1333
|
+
await snapshotWriter.save({
|
|
1334
|
+
sourceTable: snapshotTable,
|
|
1335
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1336
|
+
after: {
|
|
1337
|
+
id: 'test1',
|
|
1338
|
+
description: 'test1a'
|
|
1339
|
+
},
|
|
1340
|
+
afterReplicaId: test_utils.rid('test1')
|
|
1341
|
+
});
|
|
1342
|
+
await snapshotWriter.markAllSnapshotDone('3/1');
|
|
1343
|
+
await snapshotWriter.commit('1/1');
|
|
1344
|
+
await streamingWriter.keepalive('3/1');
|
|
1345
|
+
const cp = await bucketStorage.getCheckpoint();
|
|
1346
|
+
expect(cp.lsn).toEqual('3/1');
|
|
1347
|
+
const data = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]));
|
|
1348
|
+
expect(data).toEqual([]);
|
|
1676
1349
|
});
|
|
1677
1350
|
}
|
|
1678
1351
|
/**
|
|
@@ -1683,62 +1356,50 @@ bucket_definitions:
|
|
|
1683
1356
|
export function testChecksumBatching(config) {
|
|
1684
1357
|
const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
|
|
1685
1358
|
test('checksums for multiple buckets', async () => {
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
const factory = __addDisposableResource(env_24, await config.factory(), true);
|
|
1689
|
-
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1359
|
+
await using factory = await config.factory();
|
|
1360
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
|
|
1690
1361
|
bucket_definitions:
|
|
1691
1362
|
user:
|
|
1692
1363
|
parameters: select request.user_id() as user_id
|
|
1693
1364
|
data:
|
|
1694
1365
|
- select id, description from test where user_id = bucket.user_id
|
|
1695
1366
|
`, {
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
}
|
|
1367
|
+
storageVersion
|
|
1368
|
+
}));
|
|
1369
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
1370
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
1371
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
1372
|
+
await writer.markAllSnapshotDone('1/1');
|
|
1373
|
+
for (let u of ['u1', 'u2', 'u3', 'u4']) {
|
|
1374
|
+
for (let t of ['t1', 't2', 't3', 't4']) {
|
|
1375
|
+
const id = `${t}_${u}`;
|
|
1376
|
+
await writer.save({
|
|
1377
|
+
sourceTable,
|
|
1378
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
1379
|
+
after: {
|
|
1380
|
+
id,
|
|
1381
|
+
description: `${t} description`,
|
|
1382
|
+
user_id: u
|
|
1383
|
+
},
|
|
1384
|
+
afterReplicaId: test_utils.rid(id)
|
|
1385
|
+
});
|
|
1716
1386
|
}
|
|
1717
|
-
await writer.commit('1/1');
|
|
1718
|
-
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1719
|
-
bucketStorage.clearChecksumCache();
|
|
1720
|
-
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1721
|
-
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1722
|
-
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1723
|
-
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1724
|
-
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1725
|
-
const expected = bucketRequests.map((request, index) => ({
|
|
1726
|
-
bucket: request.bucket,
|
|
1727
|
-
count: 4,
|
|
1728
|
-
checksum: expectedChecksums[index]
|
|
1729
|
-
}));
|
|
1730
|
-
expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1731
|
-
expect(checksums).toEqual(expected);
|
|
1732
|
-
}
|
|
1733
|
-
catch (e_24) {
|
|
1734
|
-
env_24.error = e_24;
|
|
1735
|
-
env_24.hasError = true;
|
|
1736
|
-
}
|
|
1737
|
-
finally {
|
|
1738
|
-
const result_24 = __disposeResources(env_24);
|
|
1739
|
-
if (result_24)
|
|
1740
|
-
await result_24;
|
|
1741
1387
|
}
|
|
1388
|
+
await writer.commit('1/1');
|
|
1389
|
+
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
1390
|
+
bucketStorage.clearChecksumCache();
|
|
1391
|
+
const users = ['u1', 'u2', 'u3', 'u4'];
|
|
1392
|
+
const expectedChecksums = [346204588, 5261081, 134760718, -302639724];
|
|
1393
|
+
const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`));
|
|
1394
|
+
const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()];
|
|
1395
|
+
checksums.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1396
|
+
const expected = bucketRequests.map((request, index) => ({
|
|
1397
|
+
bucket: request.bucket,
|
|
1398
|
+
count: 4,
|
|
1399
|
+
checksum: expectedChecksums[index]
|
|
1400
|
+
}));
|
|
1401
|
+
expected.sort((a, b) => a.bucket.localeCompare(b.bucket));
|
|
1402
|
+
expect(checksums).toEqual(expected);
|
|
1742
1403
|
});
|
|
1743
1404
|
}
|
|
1744
1405
|
//# sourceMappingURL=register-data-storage-data-tests.js.map
|