@powersync/service-core-tests 0.0.0-dev-20260313100403 → 0.0.0-dev-20260511080634

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +61 -5
  2. package/dist/test-utils/MetricsHelper.js +1 -1
  3. package/dist/test-utils/MetricsHelper.js.map +1 -1
  4. package/dist/test-utils/StorageDataHelpers.d.ts +8 -0
  5. package/dist/test-utils/StorageDataHelpers.js +33 -0
  6. package/dist/test-utils/StorageDataHelpers.js.map +1 -0
  7. package/dist/test-utils/general-utils.d.ts +1 -1
  8. package/dist/test-utils/general-utils.js +1 -1
  9. package/dist/test-utils/stream_utils.js +2 -2
  10. package/dist/test-utils/stream_utils.js.map +1 -1
  11. package/dist/test-utils/test-utils-index.d.ts +1 -0
  12. package/dist/test-utils/test-utils-index.js +1 -0
  13. package/dist/test-utils/test-utils-index.js.map +1 -1
  14. package/dist/tests/register-compacting-tests.js +491 -627
  15. package/dist/tests/register-compacting-tests.js.map +1 -1
  16. package/dist/tests/register-data-storage-checkpoint-tests.js +192 -304
  17. package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
  18. package/dist/tests/register-data-storage-data-tests.js +1125 -1446
  19. package/dist/tests/register-data-storage-data-tests.js.map +1 -1
  20. package/dist/tests/register-data-storage-parameter-tests.js +661 -631
  21. package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
  22. package/dist/tests/register-migration-tests.js +63 -139
  23. package/dist/tests/register-migration-tests.js.map +1 -1
  24. package/dist/tests/register-parameter-compacting-tests.js +95 -171
  25. package/dist/tests/register-parameter-compacting-tests.js.map +1 -1
  26. package/dist/tests/register-report-tests.js +22 -22
  27. package/dist/tests/register-report-tests.js.map +1 -1
  28. package/dist/tests/register-sync-tests.js +965 -1155
  29. package/dist/tests/register-sync-tests.js.map +1 -1
  30. package/dist/tests/tests-index.d.ts +4 -4
  31. package/dist/tests/tests-index.js +4 -4
  32. package/dist/tests/tests-index.js.map +1 -1
  33. package/package.json +6 -6
  34. package/src/test-utils/MetricsHelper.ts +1 -1
  35. package/src/test-utils/StorageDataHelpers.ts +44 -0
  36. package/src/test-utils/general-utils.ts +1 -1
  37. package/src/test-utils/stream_utils.ts +2 -2
  38. package/src/test-utils/test-utils-index.ts +1 -0
  39. package/src/tests/register-data-storage-data-tests.ts +22 -0
  40. package/src/tests/register-data-storage-parameter-tests.ts +321 -59
  41. package/src/tests/register-parameter-compacting-tests.ts +6 -7
  42. package/src/tests/register-report-tests.ts +22 -22
  43. package/src/tests/register-sync-tests.ts +74 -0
  44. package/src/tests/tests-index.ts +4 -4
  45. package/tsconfig.tsbuildinfo +1 -1
@@ -1,61 +1,8 @@
1
- var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
- if (value !== null && value !== void 0) {
3
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
- var dispose, inner;
5
- if (async) {
6
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
- dispose = value[Symbol.asyncDispose];
8
- }
9
- if (dispose === void 0) {
10
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
- dispose = value[Symbol.dispose];
12
- if (async) inner = dispose;
13
- }
14
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
- if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
- env.stack.push({ value: value, dispose: dispose, async: async });
17
- }
18
- else if (async) {
19
- env.stack.push({ async: true });
20
- }
21
- return value;
22
- };
23
- var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
- return function (env) {
25
- function fail(e) {
26
- env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
- env.hasError = true;
28
- }
29
- var r, s = 0;
30
- function next() {
31
- while (r = env.stack.pop()) {
32
- try {
33
- if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
- if (r.dispose) {
35
- var result = r.dispose.call(r.value);
36
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
- }
38
- else s |= 1;
39
- }
40
- catch (e) {
41
- fail(e);
42
- }
43
- }
44
- if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
- if (env.hasError) throw env.error;
46
- }
47
- return next();
48
- };
49
- })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
- var e = new Error(message);
51
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
- });
53
1
  import { CURRENT_STORAGE_VERSION, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
54
- import { RequestParameters, ScopedParameterLookup } from '@powersync/service-sync-rules';
2
+ import { RequestParameters } from '@powersync/service-sync-rules';
55
3
  import { expect, test } from 'vitest';
56
4
  import * as test_utils from '../test-utils/test-utils-index.js';
57
5
  import { bucketRequest } from '../test-utils/test-utils-index.js';
58
- import { parameterLookupScope } from './util.js';
59
6
  /**
60
7
  * @example
61
8
  * ```TypeScript
@@ -69,133 +16,120 @@ import { parameterLookupScope } from './util.js';
69
16
  export function registerDataStorageParameterTests(config) {
70
17
  const generateStorageFactory = config.factory;
71
18
  const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION;
72
- const MYBUCKET_1 = parameterLookupScope('mybucket', '1');
73
19
  test('save and load parameters', async () => {
74
- const env_1 = { stack: [], error: void 0, hasError: false };
75
- try {
76
- const factory = __addDisposableResource(env_1, await generateStorageFactory(), true);
77
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
20
+ await using factory = await generateStorageFactory();
21
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
78
22
  bucket_definitions:
79
23
  mybucket:
80
24
  parameters:
81
25
  - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
82
26
  data: []
83
27
  `, {
84
- storageVersion
85
- }));
86
- const bucketStorage = factory.getInstance(syncRules);
87
- const writer = __addDisposableResource(env_1, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
88
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
89
- await writer.markAllSnapshotDone('1/1');
90
- await writer.save({
91
- sourceTable: testTable,
92
- tag: storage.SaveOperationTag.INSERT,
93
- after: {
94
- id: 't2',
95
- id1: 'user3',
96
- id2: 'user4',
97
- group_id: 'group2a'
98
- },
99
- afterReplicaId: test_utils.rid('t2')
100
- });
101
- await writer.save({
102
- sourceTable: testTable,
103
- tag: storage.SaveOperationTag.INSERT,
104
- after: {
105
- id: 't1',
106
- id1: 'user1',
107
- id2: 'user2',
108
- group_id: 'group1a'
109
- },
110
- afterReplicaId: test_utils.rid('t1')
111
- });
112
- await writer.commit('1/1');
113
- const checkpoint = await bucketStorage.getCheckpoint();
114
- const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]);
115
- expect(parameters).toEqual([
116
- {
117
- group_id: 'group1a'
118
- }
119
- ]);
120
- }
121
- catch (e_1) {
122
- env_1.error = e_1;
123
- env_1.hasError = true;
124
- }
125
- finally {
126
- const result_1 = __disposeResources(env_1);
127
- if (result_1)
128
- await result_1;
129
- }
28
+ storageVersion
29
+ }));
30
+ const bucketStorage = factory.getInstance(syncRules);
31
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
32
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
33
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
34
+ await writer.markAllSnapshotDone('1/1');
35
+ await writer.save({
36
+ sourceTable: testTable,
37
+ tag: storage.SaveOperationTag.INSERT,
38
+ after: {
39
+ id: 't2',
40
+ id1: 'user3',
41
+ id2: 'user4',
42
+ group_id: 'group2a'
43
+ },
44
+ afterReplicaId: test_utils.rid('t2')
45
+ });
46
+ await writer.save({
47
+ sourceTable: testTable,
48
+ tag: storage.SaveOperationTag.INSERT,
49
+ after: {
50
+ id: 't1',
51
+ id1: 'user1',
52
+ id2: 'user2',
53
+ group_id: 'group1a'
54
+ },
55
+ afterReplicaId: test_utils.rid('t1')
56
+ });
57
+ await writer.commit('1/1');
58
+ const checkpoint = await bucketStorage.getCheckpoint();
59
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {});
60
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
61
+ const buckets = await querier.queryDynamicBucketDescriptions({
62
+ async getParameterSets(lookups) {
63
+ expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]);
64
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
65
+ expect(parameter_sets).toEqual([{ group_id: 'group1a' }]);
66
+ return parameter_sets;
67
+ }
68
+ });
69
+ expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1a"]').bucket]);
130
70
  });
131
71
  test('it should use the latest version', async () => {
132
- const env_2 = { stack: [], error: void 0, hasError: false };
133
- try {
134
- const factory = __addDisposableResource(env_2, await generateStorageFactory(), true);
135
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
72
+ await using factory = await generateStorageFactory();
73
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
136
74
  bucket_definitions:
137
75
  mybucket:
138
76
  parameters:
139
77
  - SELECT group_id FROM test WHERE id = token_parameters.user_id
140
78
  data: []
141
79
  `, {
142
- storageVersion
143
- }));
144
- const bucketStorage = factory.getInstance(syncRules);
145
- const writer = __addDisposableResource(env_2, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
146
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
147
- await writer.markAllSnapshotDone('1/1');
148
- await writer.save({
149
- sourceTable: testTable,
150
- tag: storage.SaveOperationTag.INSERT,
151
- after: {
152
- id: 'user1',
153
- group_id: 'group1'
154
- },
155
- afterReplicaId: test_utils.rid('user1')
156
- });
157
- await writer.commit('1/1');
158
- const checkpoint1 = await bucketStorage.getCheckpoint();
159
- await writer.save({
160
- sourceTable: testTable,
161
- tag: storage.SaveOperationTag.INSERT,
162
- after: {
163
- id: 'user1',
164
- group_id: 'group2'
165
- },
166
- afterReplicaId: test_utils.rid('user1')
167
- });
168
- await writer.commit('1/2');
169
- const checkpoint2 = await bucketStorage.getCheckpoint();
170
- const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]);
171
- expect(parameters).toEqual([
172
- {
173
- group_id: 'group2'
174
- }
175
- ]);
176
- // Use the checkpoint to get older data if relevant
177
- const parameters2 = await checkpoint1.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]);
178
- expect(parameters2).toEqual([
179
- {
180
- group_id: 'group1'
181
- }
182
- ]);
183
- }
184
- catch (e_2) {
185
- env_2.error = e_2;
186
- env_2.hasError = true;
187
- }
188
- finally {
189
- const result_2 = __disposeResources(env_2);
190
- if (result_2)
191
- await result_2;
192
- }
80
+ storageVersion
81
+ }));
82
+ const bucketStorage = factory.getInstance(syncRules);
83
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
84
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
85
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
86
+ await writer.markAllSnapshotDone('1/1');
87
+ await writer.save({
88
+ sourceTable: testTable,
89
+ tag: storage.SaveOperationTag.INSERT,
90
+ after: {
91
+ id: 'user1',
92
+ group_id: 'group1'
93
+ },
94
+ afterReplicaId: test_utils.rid('user1')
95
+ });
96
+ await writer.commit('1/1');
97
+ const checkpoint1 = await bucketStorage.getCheckpoint();
98
+ await writer.save({
99
+ sourceTable: testTable,
100
+ tag: storage.SaveOperationTag.INSERT,
101
+ after: {
102
+ id: 'user1',
103
+ group_id: 'group2'
104
+ },
105
+ afterReplicaId: test_utils.rid('user1')
106
+ });
107
+ await writer.commit('1/2');
108
+ const checkpoint2 = await bucketStorage.getCheckpoint();
109
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {});
110
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
111
+ const buckets1 = await querier.queryDynamicBucketDescriptions({
112
+ async getParameterSets(lookups) {
113
+ expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]);
114
+ const parameter_sets = await checkpoint1.getParameterSets(lookups, 1000);
115
+ expect(parameter_sets).toEqual([{ group_id: 'group1' }]);
116
+ return parameter_sets;
117
+ }
118
+ });
119
+ expect(buckets1.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]);
120
+ const buckets2 = await querier.queryDynamicBucketDescriptions({
121
+ async getParameterSets(lookups) {
122
+ expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]);
123
+ const parameter_sets = await checkpoint2.getParameterSets(lookups, 1000);
124
+ expect(parameter_sets).toEqual([{ group_id: 'group2' }]);
125
+ return parameter_sets;
126
+ }
127
+ });
128
+ expect(buckets2.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group2"]').bucket]);
193
129
  });
194
130
  test('it should use the latest version after updates', async () => {
195
- const env_3 = { stack: [], error: void 0, hasError: false };
196
- try {
197
- const factory = __addDisposableResource(env_3, await generateStorageFactory(), true);
198
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
131
+ await using factory = await generateStorageFactory();
132
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
199
133
  bucket_definitions:
200
134
  mybucket:
201
135
  parameters:
@@ -204,186 +138,172 @@ bucket_definitions:
204
138
  WHERE list_id IN token_parameters.list_id
205
139
  data: []
206
140
  `, { storageVersion }));
207
- const bucketStorage = factory.getInstance(syncRules);
208
- const writer = __addDisposableResource(env_3, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
209
- const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config);
210
- await writer.markAllSnapshotDone('1/1');
211
- // Create two todos which initially belong to different lists
212
- await writer.save({
213
- sourceTable: table,
214
- tag: storage.SaveOperationTag.INSERT,
215
- after: {
216
- id: 'todo1',
217
- list_id: 'list1'
218
- },
219
- afterReplicaId: test_utils.rid('todo1')
220
- });
221
- await writer.save({
222
- sourceTable: table,
223
- tag: storage.SaveOperationTag.INSERT,
224
- after: {
225
- id: 'todo2',
226
- list_id: 'list2'
227
- },
228
- afterReplicaId: test_utils.rid('todo2')
229
- });
230
- await writer.commit('1/1');
231
- // Update the second todo item to now belong to list 1
232
- await writer.save({
233
- sourceTable: table,
234
- tag: storage.SaveOperationTag.UPDATE,
235
- after: {
236
- id: 'todo2',
237
- list_id: 'list1'
238
- },
239
- afterReplicaId: test_utils.rid('todo2')
240
- });
241
- await writer.commit('1/1');
242
- // We specifically request the todo_ids for both lists.
243
- // There removal operation for the association of `list2`::`todo2` should not interfere with the new
244
- // association of `list1`::`todo2`
245
- const checkpoint = await bucketStorage.getCheckpoint();
246
- const parameters = await checkpoint.getParameterSets([
247
- ScopedParameterLookup.direct(MYBUCKET_1, ['list1']),
248
- ScopedParameterLookup.direct(MYBUCKET_1, ['list2'])
249
- ]);
250
- expect(parameters.sort((a, b) => a.todo_id.localeCompare(b.todo_id))).toEqual([
251
- {
252
- todo_id: 'todo1'
253
- },
254
- {
255
- todo_id: 'todo2'
256
- }
257
- ]);
258
- }
259
- catch (e_3) {
260
- env_3.error = e_3;
261
- env_3.hasError = true;
262
- }
263
- finally {
264
- const result_3 = __disposeResources(env_3);
265
- if (result_3)
266
- await result_3;
267
- }
141
+ const bucketStorage = factory.getInstance(syncRules);
142
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
143
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
144
+ const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config);
145
+ await writer.markAllSnapshotDone('1/1');
146
+ // Create two todos which initially belong to different lists
147
+ await writer.save({
148
+ sourceTable: table,
149
+ tag: storage.SaveOperationTag.INSERT,
150
+ after: {
151
+ id: 'todo1',
152
+ list_id: 'list1'
153
+ },
154
+ afterReplicaId: test_utils.rid('todo1')
155
+ });
156
+ await writer.save({
157
+ sourceTable: table,
158
+ tag: storage.SaveOperationTag.INSERT,
159
+ after: {
160
+ id: 'todo2',
161
+ list_id: 'list2'
162
+ },
163
+ afterReplicaId: test_utils.rid('todo2')
164
+ });
165
+ await writer.commit('1/1');
166
+ // Update the second todo item to now belong to list 1
167
+ await writer.save({
168
+ sourceTable: table,
169
+ tag: storage.SaveOperationTag.UPDATE,
170
+ after: {
171
+ id: 'todo2',
172
+ list_id: 'list1'
173
+ },
174
+ afterReplicaId: test_utils.rid('todo2')
175
+ });
176
+ await writer.commit('1/1');
177
+ // We specifically request the todo_ids for both lists.
178
+ // There removal operation for the association of `list2`::`todo2` should not interfere with the new
179
+ // association of `list1`::`todo2`
180
+ const checkpoint = await bucketStorage.getCheckpoint();
181
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { list_id: ['list1', 'list2'] } }), {});
182
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
183
+ const buckets = await querier.queryDynamicBucketDescriptions({
184
+ async getParameterSets(lookups) {
185
+ expect(lookups.map((l) => JSON.stringify(l.indexKey)).sort()).toEqual(['["list1"]', '["list2"]']);
186
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
187
+ expect(parameter_sets.sort((a, b) => a.todo_id.localeCompare(b.todo_id))).toEqual([
188
+ { todo_id: 'todo1' },
189
+ { todo_id: 'todo2' }
190
+ ]);
191
+ return parameter_sets;
192
+ }
193
+ });
194
+ expect(buckets.map((b) => b.bucket).sort()).toEqual([
195
+ bucketRequest(syncRules, 'mybucket["todo1"]').bucket,
196
+ bucketRequest(syncRules, 'mybucket["todo2"]').bucket
197
+ ]);
268
198
  });
269
199
  test('save and load parameters with different number types', async () => {
270
- const env_4 = { stack: [], error: void 0, hasError: false };
271
- try {
272
- const factory = __addDisposableResource(env_4, await generateStorageFactory(), true);
273
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
200
+ await using factory = await generateStorageFactory();
201
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
274
202
  bucket_definitions:
275
203
  mybucket:
276
204
  parameters:
277
205
  - SELECT group_id FROM test WHERE n1 = token_parameters.n1 and f2 = token_parameters.f2 and f3 = token_parameters.f3
278
206
  data: []
279
207
  `, {
280
- storageVersion
281
- }));
282
- const bucketStorage = factory.getInstance(syncRules);
283
- const writer = __addDisposableResource(env_4, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
284
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
285
- await writer.markAllSnapshotDone('1/1');
286
- await writer.save({
287
- sourceTable: testTable,
288
- tag: storage.SaveOperationTag.INSERT,
289
- after: {
290
- id: 't1',
291
- group_id: 'group1',
292
- n1: 314n,
293
- f2: 314,
294
- f3: 3.14
295
- },
296
- afterReplicaId: test_utils.rid('t1')
208
+ storageVersion
209
+ }));
210
+ const bucketStorage = factory.getInstance(syncRules);
211
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
212
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
213
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
214
+ await writer.markAllSnapshotDone('1/1');
215
+ await writer.save({
216
+ sourceTable: testTable,
217
+ tag: storage.SaveOperationTag.INSERT,
218
+ after: {
219
+ id: 't1',
220
+ group_id: 'group1',
221
+ n1: 314n,
222
+ f2: 314,
223
+ f3: 3.14
224
+ },
225
+ afterReplicaId: test_utils.rid('t1')
226
+ });
227
+ await writer.commit('1/1');
228
+ const checkpoint = await bucketStorage.getCheckpoint();
229
+ const testQuery = async (jwtParameters, expectedParameterSets) => {
230
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: jwtParameters }), {});
231
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
232
+ return await querier.queryDynamicBucketDescriptions({
233
+ async getParameterSets(lookups) {
234
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
235
+ expect(parameter_sets).toEqual(expectedParameterSets);
236
+ return parameter_sets;
237
+ }
297
238
  });
298
- await writer.commit('1/1');
299
- const TEST_PARAMS = { group_id: 'group1' };
300
- const checkpoint = await bucketStorage.getCheckpoint();
301
- const parameters1 = await checkpoint.getParameterSets([
302
- ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3.14])
303
- ]);
304
- expect(parameters1).toEqual([TEST_PARAMS]);
305
- const parameters2 = await checkpoint.getParameterSets([
306
- ScopedParameterLookup.direct(MYBUCKET_1, [314, 314n, 3.14])
307
- ]);
308
- expect(parameters2).toEqual([TEST_PARAMS]);
309
- const parameters3 = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3])]);
310
- expect(parameters3).toEqual([]);
311
- }
312
- catch (e_4) {
313
- env_4.error = e_4;
314
- env_4.hasError = true;
315
- }
316
- finally {
317
- const result_4 = __disposeResources(env_4);
318
- if (result_4)
319
- await result_4;
320
- }
239
+ };
240
+ expect(await testQuery({ n1: 314n, f2: 314, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([
241
+ { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket }
242
+ ]);
243
+ expect(await testQuery({ n1: 314, f2: 314n, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([
244
+ { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket }
245
+ ]);
246
+ expect(await testQuery({ n1: 314n, f2: 314, f3: 3 }, [])).toEqual([]);
321
247
  });
322
248
  test('save and load parameters with large numbers', async () => {
323
- const env_5 = { stack: [], error: void 0, hasError: false };
324
- try {
325
- // This ensures serialization / deserialization of "current_data" is done correctly.
326
- // This specific case tested here cannot happen with postgres in practice, but we still
327
- // test this to ensure correct deserialization.
328
- const factory = __addDisposableResource(env_5, await generateStorageFactory(), true);
329
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
249
+ // This ensures serialization / deserialization of "current_data" is done correctly.
250
+ // This specific case tested here cannot happen with postgres in practice, but we still
251
+ // test this to ensure correct deserialization.
252
+ await using factory = await generateStorageFactory();
253
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
330
254
  bucket_definitions:
331
255
  mybucket:
332
256
  parameters:
333
257
  - SELECT group_id FROM test WHERE n1 = token_parameters.n1
334
258
  data: []
335
259
  `, {
336
- storageVersion
337
- }));
338
- const bucketStorage = factory.getInstance(syncRules);
339
- const writer = __addDisposableResource(env_5, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
340
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
341
- await writer.markAllSnapshotDone('1/1');
342
- await writer.save({
343
- sourceTable: testTable,
344
- tag: storage.SaveOperationTag.INSERT,
345
- after: {
346
- id: 't1',
347
- group_id: 'group1',
348
- n1: 1152921504606846976n // 2^60
349
- },
350
- afterReplicaId: test_utils.rid('t1')
351
- });
352
- await writer.save({
353
- sourceTable: testTable,
354
- tag: storage.SaveOperationTag.UPDATE,
355
- after: {
356
- id: 't1',
357
- group_id: 'group1',
358
- // Simulate a TOAST value, even though it can't happen for values like this
359
- // in practice.
360
- n1: undefined
361
- },
362
- afterReplicaId: test_utils.rid('t1')
363
- });
364
- await writer.commit('1/1');
365
- const TEST_PARAMS = { group_id: 'group1' };
366
- const checkpoint = await bucketStorage.getCheckpoint();
367
- const parameters1 = await checkpoint.getParameterSets([
368
- ScopedParameterLookup.direct(MYBUCKET_1, [1152921504606846976n])
369
- ]);
370
- expect(parameters1).toEqual([TEST_PARAMS]);
371
- }
372
- catch (e_5) {
373
- env_5.error = e_5;
374
- env_5.hasError = true;
375
- }
376
- finally {
377
- const result_5 = __disposeResources(env_5);
378
- if (result_5)
379
- await result_5;
380
- }
260
+ storageVersion
261
+ }));
262
+ const bucketStorage = factory.getInstance(syncRules);
263
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
264
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
265
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
266
+ await writer.markAllSnapshotDone('1/1');
267
+ await writer.save({
268
+ sourceTable: testTable,
269
+ tag: storage.SaveOperationTag.INSERT,
270
+ after: {
271
+ id: 't1',
272
+ group_id: 'group1',
273
+ n1: 1152921504606846976n // 2^60
274
+ },
275
+ afterReplicaId: test_utils.rid('t1')
276
+ });
277
+ await writer.save({
278
+ sourceTable: testTable,
279
+ tag: storage.SaveOperationTag.UPDATE,
280
+ after: {
281
+ id: 't1',
282
+ group_id: 'group1',
283
+ // Simulate a TOAST value, even though it can't happen for values like this
284
+ // in practice.
285
+ n1: undefined
286
+ },
287
+ afterReplicaId: test_utils.rid('t1')
288
+ });
289
+ await writer.commit('1/1');
290
+ const checkpoint = await bucketStorage.getCheckpoint();
291
+ const n1 = 1152921504606846976n;
292
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { n1 } }), {});
293
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
294
+ const buckets = await querier.queryDynamicBucketDescriptions({
295
+ getParameterSets: async (lookups) => {
296
+ expect(lookups.map((l) => l.indexKey)).toEqual([[n1]]);
297
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
298
+ expect(parameter_sets).toEqual([{ group_id: 'group1' }]);
299
+ return parameter_sets;
300
+ }
301
+ });
302
+ expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]);
381
303
  });
382
304
  test('save and load parameters with workspaceId', async () => {
383
- const env_6 = { stack: [], error: void 0, hasError: false };
384
- try {
385
- const factory = __addDisposableResource(env_6, await generateStorageFactory(), true);
386
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
305
+ await using factory = await generateStorageFactory();
306
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
387
307
  bucket_definitions:
388
308
  by_workspace:
389
309
  parameters:
@@ -391,58 +311,46 @@ bucket_definitions:
391
311
  workspace."userId" = token_parameters.user_id
392
312
  data: []
393
313
  `, {
394
- storageVersion
395
- }));
396
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
397
- const bucketStorage = factory.getInstance(syncRules);
398
- const writer = __addDisposableResource(env_6, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
399
- const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config);
400
- await writer.markAllSnapshotDone('1/1');
401
- await writer.save({
402
- sourceTable: workspaceTable,
403
- tag: storage.SaveOperationTag.INSERT,
404
- after: {
405
- id: 'workspace1',
406
- userId: 'u1'
407
- },
408
- afterReplicaId: test_utils.rid('workspace1')
409
- });
410
- await writer.commit('1/1');
411
- const checkpoint = await bucketStorage.getCheckpoint();
412
- const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {});
413
- const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
414
- const buckets = await querier.queryDynamicBucketDescriptions({
415
- async getParameterSets(lookups) {
416
- expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), ['u1'])]);
417
- const parameter_sets = await checkpoint.getParameterSets(lookups);
418
- expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
419
- return parameter_sets;
420
- }
421
- });
422
- expect(buckets).toEqual([
423
- {
424
- bucket: bucketRequest(syncRules, 'by_workspace["workspace1"]').bucket,
425
- priority: 3,
426
- definition: 'by_workspace',
427
- inclusion_reasons: ['default']
428
- }
429
- ]);
430
- }
431
- catch (e_6) {
432
- env_6.error = e_6;
433
- env_6.hasError = true;
434
- }
435
- finally {
436
- const result_6 = __disposeResources(env_6);
437
- if (result_6)
438
- await result_6;
439
- }
314
+ storageVersion
315
+ }));
316
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
317
+ const bucketStorage = factory.getInstance(syncRules);
318
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
319
+ const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config);
320
+ await writer.markAllSnapshotDone('1/1');
321
+ await writer.save({
322
+ sourceTable: workspaceTable,
323
+ tag: storage.SaveOperationTag.INSERT,
324
+ after: {
325
+ id: 'workspace1',
326
+ userId: 'u1'
327
+ },
328
+ afterReplicaId: test_utils.rid('workspace1')
329
+ });
330
+ await writer.commit('1/1');
331
+ const checkpoint = await bucketStorage.getCheckpoint();
332
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {});
333
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
334
+ const buckets = await querier.queryDynamicBucketDescriptions({
335
+ async getParameterSets(lookups) {
336
+ expect(lookups.map((l) => l.indexKey)).toEqual([['u1']]);
337
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
338
+ expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]);
339
+ return parameter_sets;
340
+ }
341
+ });
342
+ expect(buckets).toEqual([
343
+ {
344
+ bucket: bucketRequest(syncRules, 'by_workspace["workspace1"]').bucket,
345
+ priority: 3,
346
+ definition: 'by_workspace',
347
+ inclusion_reasons: ['default']
348
+ }
349
+ ]);
440
350
  });
441
351
  test('save and load parameters with dynamic global buckets', async () => {
442
- const env_7 = { stack: [], error: void 0, hasError: false };
443
- try {
444
- const factory = __addDisposableResource(env_7, await generateStorageFactory(), true);
445
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
352
+ await using factory = await generateStorageFactory();
353
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
446
354
  bucket_definitions:
447
355
  by_public_workspace:
448
356
  parameters:
@@ -450,84 +358,72 @@ bucket_definitions:
450
358
  workspace.visibility = 'public'
451
359
  data: []
452
360
  `, {
453
- storageVersion
454
- }));
455
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
456
- const bucketStorage = factory.getInstance(syncRules);
457
- const writer = __addDisposableResource(env_7, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
458
- const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config);
459
- await writer.markAllSnapshotDone('1/1');
460
- await writer.save({
461
- sourceTable: workspaceTable,
462
- tag: storage.SaveOperationTag.INSERT,
463
- after: {
464
- id: 'workspace1',
465
- visibility: 'public'
466
- },
467
- afterReplicaId: test_utils.rid('workspace1')
468
- });
469
- await writer.save({
470
- sourceTable: workspaceTable,
471
- tag: storage.SaveOperationTag.INSERT,
472
- after: {
473
- id: 'workspace2',
474
- visibility: 'private'
475
- },
476
- afterReplicaId: test_utils.rid('workspace2')
477
- });
478
- await writer.save({
479
- sourceTable: workspaceTable,
480
- tag: storage.SaveOperationTag.INSERT,
481
- after: {
482
- id: 'workspace3',
483
- visibility: 'public'
484
- },
485
- afterReplicaId: test_utils.rid('workspace3')
486
- });
487
- await writer.commit('1/1');
488
- const checkpoint = await bucketStorage.getCheckpoint();
489
- const parameters = new RequestParameters(new JwtPayload({ sub: 'unknown' }), {});
490
- const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
491
- const buckets = await querier.queryDynamicBucketDescriptions({
492
- async getParameterSets(lookups) {
493
- expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_public_workspace', '1'), [])]);
494
- const parameter_sets = await checkpoint.getParameterSets(lookups);
495
- parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
496
- expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
497
- return parameter_sets;
498
- }
499
- });
500
- buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
501
- expect(buckets).toEqual([
502
- {
503
- bucket: bucketRequest(syncRules, 'by_public_workspace["workspace1"]').bucket,
504
- priority: 3,
505
- definition: 'by_public_workspace',
506
- inclusion_reasons: ['default']
507
- },
508
- {
509
- bucket: bucketRequest(syncRules, 'by_public_workspace["workspace3"]').bucket,
510
- priority: 3,
511
- definition: 'by_public_workspace',
512
- inclusion_reasons: ['default']
513
- }
514
- ]);
515
- }
516
- catch (e_7) {
517
- env_7.error = e_7;
518
- env_7.hasError = true;
519
- }
520
- finally {
521
- const result_7 = __disposeResources(env_7);
522
- if (result_7)
523
- await result_7;
524
- }
361
+ storageVersion
362
+ }));
363
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
364
+ const bucketStorage = factory.getInstance(syncRules);
365
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
366
+ const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config);
367
+ await writer.markAllSnapshotDone('1/1');
368
+ await writer.save({
369
+ sourceTable: workspaceTable,
370
+ tag: storage.SaveOperationTag.INSERT,
371
+ after: {
372
+ id: 'workspace1',
373
+ visibility: 'public'
374
+ },
375
+ afterReplicaId: test_utils.rid('workspace1')
376
+ });
377
+ await writer.save({
378
+ sourceTable: workspaceTable,
379
+ tag: storage.SaveOperationTag.INSERT,
380
+ after: {
381
+ id: 'workspace2',
382
+ visibility: 'private'
383
+ },
384
+ afterReplicaId: test_utils.rid('workspace2')
385
+ });
386
+ await writer.save({
387
+ sourceTable: workspaceTable,
388
+ tag: storage.SaveOperationTag.INSERT,
389
+ after: {
390
+ id: 'workspace3',
391
+ visibility: 'public'
392
+ },
393
+ afterReplicaId: test_utils.rid('workspace3')
394
+ });
395
+ await writer.commit('1/1');
396
+ const checkpoint = await bucketStorage.getCheckpoint();
397
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'unknown' }), {});
398
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
399
+ const buckets = await querier.queryDynamicBucketDescriptions({
400
+ async getParameterSets(lookups) {
401
+ expect(lookups.map((l) => l.indexKey)).toEqual([[]]);
402
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
403
+ parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
404
+ expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
405
+ return parameter_sets;
406
+ }
407
+ });
408
+ buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
409
+ expect(buckets).toEqual([
410
+ {
411
+ bucket: bucketRequest(syncRules, 'by_public_workspace["workspace1"]').bucket,
412
+ priority: 3,
413
+ definition: 'by_public_workspace',
414
+ inclusion_reasons: ['default']
415
+ },
416
+ {
417
+ bucket: bucketRequest(syncRules, 'by_public_workspace["workspace3"]').bucket,
418
+ priority: 3,
419
+ definition: 'by_public_workspace',
420
+ inclusion_reasons: ['default']
421
+ }
422
+ ]);
525
423
  });
526
424
  test('multiple parameter queries', async () => {
527
- const env_8 = { stack: [], error: void 0, hasError: false };
528
- try {
529
- const factory = __addDisposableResource(env_8, await generateStorageFactory(), true);
530
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
425
+ await using factory = await generateStorageFactory();
426
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
531
427
  bucket_definitions:
532
428
  by_workspace:
533
429
  parameters:
@@ -537,138 +433,122 @@ bucket_definitions:
537
433
  workspace.user_id = token_parameters.user_id
538
434
  data: []
539
435
  `, {
540
- storageVersion
541
- }));
542
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
543
- const bucketStorage = factory.getInstance(syncRules);
544
- const writer = __addDisposableResource(env_8, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
545
- const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config);
546
- await writer.markAllSnapshotDone('1/1');
547
- await writer.save({
548
- sourceTable: workspaceTable,
549
- tag: storage.SaveOperationTag.INSERT,
550
- after: {
551
- id: 'workspace1',
552
- visibility: 'public'
553
- },
554
- afterReplicaId: test_utils.rid('workspace1')
555
- });
556
- await writer.save({
557
- sourceTable: workspaceTable,
558
- tag: storage.SaveOperationTag.INSERT,
559
- after: {
560
- id: 'workspace2',
561
- visibility: 'private'
562
- },
563
- afterReplicaId: test_utils.rid('workspace2')
564
- });
565
- await writer.save({
566
- sourceTable: workspaceTable,
567
- tag: storage.SaveOperationTag.INSERT,
568
- after: {
569
- id: 'workspace3',
570
- user_id: 'u1',
571
- visibility: 'private'
572
- },
573
- afterReplicaId: test_utils.rid('workspace3')
574
- });
575
- await writer.save({
576
- sourceTable: workspaceTable,
577
- tag: storage.SaveOperationTag.INSERT,
578
- after: {
579
- id: 'workspace4',
580
- user_id: 'u2',
581
- visibility: 'private'
582
- },
583
- afterReplicaId: test_utils.rid('workspace4')
584
- });
585
- await writer.commit('1/1');
586
- const checkpoint = await bucketStorage.getCheckpoint();
587
- const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {});
588
- // Test intermediate values - could be moved to sync_rules.test.ts
589
- const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
590
- // Test final values - the important part
591
- const foundLookups = [];
592
- const parameter_sets = [];
593
- const buckets = (await querier.queryDynamicBucketDescriptions({
594
- async getParameterSets(lookups) {
595
- foundLookups.push(...lookups);
596
- const output = await checkpoint.getParameterSets(lookups);
597
- parameter_sets.push(...output);
598
- return output;
599
- }
600
- })).map((e) => e.bucket);
601
- expect(foundLookups).toEqual([
602
- ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), []),
603
- ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '2'), ['u1'])
604
- ]);
605
- parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
606
- expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
607
- buckets.sort();
608
- expect(buckets).toEqual([
609
- bucketRequest(syncRules, 'by_workspace["workspace1"]').bucket,
610
- bucketRequest(syncRules, 'by_workspace["workspace3"]').bucket
611
- ]);
612
- }
613
- catch (e_8) {
614
- env_8.error = e_8;
615
- env_8.hasError = true;
616
- }
617
- finally {
618
- const result_8 = __disposeResources(env_8);
619
- if (result_8)
620
- await result_8;
621
- }
436
+ storageVersion
437
+ }));
438
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
439
+ const bucketStorage = factory.getInstance(syncRules);
440
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
441
+ const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config);
442
+ await writer.markAllSnapshotDone('1/1');
443
+ await writer.save({
444
+ sourceTable: workspaceTable,
445
+ tag: storage.SaveOperationTag.INSERT,
446
+ after: {
447
+ id: 'workspace1',
448
+ visibility: 'public'
449
+ },
450
+ afterReplicaId: test_utils.rid('workspace1')
451
+ });
452
+ await writer.save({
453
+ sourceTable: workspaceTable,
454
+ tag: storage.SaveOperationTag.INSERT,
455
+ after: {
456
+ id: 'workspace2',
457
+ visibility: 'private'
458
+ },
459
+ afterReplicaId: test_utils.rid('workspace2')
460
+ });
461
+ await writer.save({
462
+ sourceTable: workspaceTable,
463
+ tag: storage.SaveOperationTag.INSERT,
464
+ after: {
465
+ id: 'workspace3',
466
+ user_id: 'u1',
467
+ visibility: 'private'
468
+ },
469
+ afterReplicaId: test_utils.rid('workspace3')
470
+ });
471
+ await writer.save({
472
+ sourceTable: workspaceTable,
473
+ tag: storage.SaveOperationTag.INSERT,
474
+ after: {
475
+ id: 'workspace4',
476
+ user_id: 'u2',
477
+ visibility: 'private'
478
+ },
479
+ afterReplicaId: test_utils.rid('workspace4')
480
+ });
481
+ await writer.commit('1/1');
482
+ const checkpoint = await bucketStorage.getCheckpoint();
483
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {});
484
+ // Test intermediate values - could be moved to sync_rules.test.ts
485
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
486
+ // Test final values - the important part
487
+ const foundLookups = [];
488
+ const parameter_sets = [];
489
+ const buckets = (await querier.queryDynamicBucketDescriptions({
490
+ async getParameterSets(lookups) {
491
+ foundLookups.push(...lookups);
492
+ const output = await checkpoint.getParameterSets(lookups, 1000);
493
+ parameter_sets.push(...output);
494
+ return output;
495
+ }
496
+ })).map((e) => e.bucket);
497
+ // Not testing the scope anymore - the exact format depends on storage version
498
+ expect(foundLookups.map((l) => l.indexKey)).toEqual([[], ['u1']]);
499
+ parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
500
+ expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
501
+ buckets.sort();
502
+ expect(buckets).toEqual([
503
+ bucketRequest(syncRules, 'by_workspace["workspace1"]').bucket,
504
+ bucketRequest(syncRules, 'by_workspace["workspace3"]').bucket
505
+ ]);
622
506
  });
623
507
  test('truncate parameters', async () => {
624
- const env_9 = { stack: [], error: void 0, hasError: false };
625
- try {
626
- const factory = __addDisposableResource(env_9, await generateStorageFactory(), true);
627
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
508
+ await using factory = await generateStorageFactory();
509
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
628
510
  bucket_definitions:
629
511
  mybucket:
630
512
  parameters:
631
513
  - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
632
514
  data: []
633
515
  `, {
634
- storageVersion
635
- }));
636
- const bucketStorage = factory.getInstance(syncRules);
637
- const writer = __addDisposableResource(env_9, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
638
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
639
- await writer.markAllSnapshotDone('1/1');
640
- await writer.save({
641
- sourceTable: testTable,
642
- tag: storage.SaveOperationTag.INSERT,
643
- after: {
644
- id: 't2',
645
- id1: 'user3',
646
- id2: 'user4',
647
- group_id: 'group2a'
648
- },
649
- afterReplicaId: test_utils.rid('t2')
650
- });
651
- await writer.truncate([testTable]);
652
- await writer.flush();
653
- const checkpoint = await bucketStorage.getCheckpoint();
654
- const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]);
655
- expect(parameters).toEqual([]);
656
- }
657
- catch (e_9) {
658
- env_9.error = e_9;
659
- env_9.hasError = true;
660
- }
661
- finally {
662
- const result_9 = __disposeResources(env_9);
663
- if (result_9)
664
- await result_9;
665
- }
516
+ storageVersion
517
+ }));
518
+ const bucketStorage = factory.getInstance(syncRules);
519
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
520
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
521
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
522
+ await writer.markAllSnapshotDone('1/1');
523
+ await writer.save({
524
+ sourceTable: testTable,
525
+ tag: storage.SaveOperationTag.INSERT,
526
+ after: {
527
+ id: 't2',
528
+ id1: 'user3',
529
+ id2: 'user4',
530
+ group_id: 'group2a'
531
+ },
532
+ afterReplicaId: test_utils.rid('t2')
533
+ });
534
+ await writer.truncate([testTable]);
535
+ await writer.flush();
536
+ const checkpoint = await bucketStorage.getCheckpoint();
537
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {});
538
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
539
+ const buckets = await querier.queryDynamicBucketDescriptions({
540
+ async getParameterSets(lookups) {
541
+ expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]);
542
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
543
+ expect(parameter_sets).toEqual([]);
544
+ return parameter_sets;
545
+ }
546
+ });
547
+ expect(buckets).toEqual([]);
666
548
  });
667
549
  test('invalidate cached parsed sync rules', async () => {
668
- const env_10 = { stack: [], error: void 0, hasError: false };
669
- try {
670
- const bucketStorageFactory = __addDisposableResource(env_10, await generateStorageFactory(), true);
671
- const syncRules = await bucketStorageFactory.updateSyncRules(updateSyncRulesFromYaml(`
550
+ await using bucketStorageFactory = await generateStorageFactory();
551
+ const syncRules = await bucketStorageFactory.updateSyncRules(updateSyncRulesFromYaml(`
672
552
  bucket_definitions:
673
553
  by_workspace:
674
554
  parameters:
@@ -676,40 +556,28 @@ bucket_definitions:
676
556
  workspace."userId" = token_parameters.user_id
677
557
  data: []
678
558
  `, {
679
- storageVersion
680
- }));
681
- const syncBucketStorage = bucketStorageFactory.getInstance(syncRules);
682
- const parsedSchema1 = syncBucketStorage.getParsedSyncRules({
683
- defaultSchema: 'public'
684
- });
685
- const parsedSchema2 = syncBucketStorage.getParsedSyncRules({
686
- defaultSchema: 'public'
687
- });
688
- // These should be cached, this will be the same instance
689
- expect(parsedSchema2).equals(parsedSchema1);
690
- expect(parsedSchema1.getSourceTables()[0].schema).equals('public');
691
- const parsedSchema3 = syncBucketStorage.getParsedSyncRules({
692
- defaultSchema: 'databasename'
693
- });
694
- // The cache should not be used
695
- expect(parsedSchema3).not.equals(parsedSchema2);
696
- expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename');
697
- }
698
- catch (e_10) {
699
- env_10.error = e_10;
700
- env_10.hasError = true;
701
- }
702
- finally {
703
- const result_10 = __disposeResources(env_10);
704
- if (result_10)
705
- await result_10;
706
- }
559
+ storageVersion
560
+ }));
561
+ const syncBucketStorage = bucketStorageFactory.getInstance(syncRules);
562
+ const parsedSchema1 = syncBucketStorage.getParsedSyncRules({
563
+ defaultSchema: 'public'
564
+ });
565
+ const parsedSchema2 = syncBucketStorage.getParsedSyncRules({
566
+ defaultSchema: 'public'
567
+ });
568
+ // These should be cached, this will be the same instance
569
+ expect(parsedSchema2).equals(parsedSchema1);
570
+ expect(parsedSchema1.getSourceTables()[0].schema).equals('public');
571
+ const parsedSchema3 = syncBucketStorage.getParsedSyncRules({
572
+ defaultSchema: 'databasename'
573
+ });
574
+ // The cache should not be used
575
+ expect(parsedSchema3).not.equals(parsedSchema2);
576
+ expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename');
707
577
  });
708
578
  test('sync streams smoke test', async () => {
709
- const env_11 = { stack: [], error: void 0, hasError: false };
710
- try {
711
- const factory = __addDisposableResource(env_11, await generateStorageFactory(), true);
712
- const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
579
+ await using factory = await generateStorageFactory();
580
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
713
581
  config:
714
582
  edition: 3
715
583
 
@@ -719,39 +587,201 @@ streams:
719
587
  SELECT data.* FROM test AS data, test AS param
720
588
  WHERE data.foo = param.bar AND param.baz = auth.user_id()
721
589
  `));
722
- const bucketStorage = factory.getInstance(syncRules);
723
- const writer = __addDisposableResource(env_11, await bucketStorage.createWriter(test_utils.BATCH_OPTIONS), true);
724
- const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
725
- await writer.markAllSnapshotDone('1/1');
590
+ const bucketStorage = factory.getInstance(syncRules);
591
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
592
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
593
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
594
+ await writer.markAllSnapshotDone('1/1');
595
+ await writer.save({
596
+ sourceTable: testTable,
597
+ tag: storage.SaveOperationTag.INSERT,
598
+ after: {
599
+ baz: 'baz',
600
+ bar: 'bar'
601
+ },
602
+ afterReplicaId: test_utils.rid('t1')
603
+ });
604
+ await writer.commit('1/1');
605
+ const checkpoint = await bucketStorage.getCheckpoint();
606
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'baz' }), {});
607
+ const querier = sync_rules.getBucketParameterQuerier({
608
+ ...test_utils.querierOptions(parameters),
609
+ streams: {
610
+ stream: [
611
+ {
612
+ priorityOverride: null,
613
+ parameters: null,
614
+ opaque_id: 123
615
+ }
616
+ ]
617
+ }
618
+ }).querier;
619
+ const buckets = await querier.queryDynamicBucketDescriptions({
620
+ async getParameterSets(lookups) {
621
+ expect(lookups.map((l) => l.indexKey)).toEqual([['baz']]);
622
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
623
+ expect(parameter_sets).toEqual([{ '0': 'bar' }]);
624
+ return parameter_sets;
625
+ }
626
+ });
627
+ console.log('whatabuckets', buckets);
628
+ expect(buckets).toHaveLength(1);
629
+ expect(buckets).toMatchObject([
630
+ {
631
+ bucket: expect.stringMatching(/stream.*\["bar"\]$/),
632
+ definition: 'stream',
633
+ inclusion_reasons: [{ subscription: 123 }],
634
+ priority: 3
635
+ }
636
+ ]);
637
+ });
638
+ test('respects parameter limit', async () => {
639
+ await using factory = await generateStorageFactory();
640
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
641
+ config:
642
+ edition: 3
643
+
644
+ streams:
645
+ a:
646
+ auto_subscribe: true
647
+ query: SELECT * FROM a WHERE id IN (SELECT id FROM b)
648
+ `, {
649
+ storageVersion
650
+ }));
651
+ const bucketStorage = factory.getInstance(syncRules);
652
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
653
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
654
+ const testTable = await test_utils.resolveTestTable(writer, 'b', ['id'], config);
655
+ await writer.markAllSnapshotDone('1/1');
656
+ for (let i = 0; i < 10; i++) {
726
657
  await writer.save({
727
658
  sourceTable: testTable,
728
659
  tag: storage.SaveOperationTag.INSERT,
729
660
  after: {
730
- baz: 'baz',
731
- bar: 'bar'
661
+ id: `t${i}`
732
662
  },
733
- afterReplicaId: test_utils.rid('t1')
663
+ afterReplicaId: test_utils.rid(`t${i}`)
734
664
  });
735
- await writer.commit('1/1');
736
- const checkpoint = await bucketStorage.getCheckpoint();
737
- const parameters = await checkpoint.getParameterSets([
738
- ScopedParameterLookup.direct(parameterLookupScope('lookup', '0'), ['baz'])
739
- ]);
740
- expect(parameters).toEqual([
741
- {
742
- '0': 'bar'
743
- }
744
- ]);
745
- }
746
- catch (e_11) {
747
- env_11.error = e_11;
748
- env_11.hasError = true;
749
- }
750
- finally {
751
- const result_11 = __disposeResources(env_11);
752
- if (result_11)
753
- await result_11;
754
665
  }
666
+ await writer.commit('1/1');
667
+ const checkpoint = await bucketStorage.getCheckpoint();
668
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'u' }), {});
669
+ const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
670
+ await expect(querier.queryDynamicBucketDescriptions({
671
+ async getParameterSets(lookups) {
672
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 5);
673
+ return parameter_sets;
674
+ }
675
+ })).rejects.toThrow('Too many parameter results (limit was 5)');
676
+ });
677
+ test('sync streams store multiple parameter outputs for a single source row and lookup', async () => {
678
+ await using factory = await generateStorageFactory();
679
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(`
680
+ config:
681
+ edition: 3
682
+ streams:
683
+ chat:
684
+ auto_subscribe: true
685
+ query: |
686
+ SELECT a.*
687
+ FROM a, b, json_each(b.x) x, json_each(b.y) y
688
+ WHERE a.x = x.value AND y.value = auth.user_id()
689
+ `));
690
+ const bucketStorage = factory.getInstance(syncRules);
691
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
692
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
693
+ const tableB = await test_utils.resolveTestTable(writer, 'b', ['id'], config);
694
+ const firstState = {
695
+ id: 'id0',
696
+ x: JSON.stringify(['x1', 'x2']),
697
+ y: JSON.stringify(['y1', 'y2'])
698
+ };
699
+ const secondState = {
700
+ id: 'id0',
701
+ x: JSON.stringify(['x2']),
702
+ y: JSON.stringify(['y1', 'y2'])
703
+ };
704
+ const thirdState = {
705
+ id: 'id0',
706
+ x: JSON.stringify(['x2']),
707
+ y: JSON.stringify(['y2'])
708
+ };
709
+ const replicaId = test_utils.rid('id0');
710
+ await writer.markAllSnapshotDone('1/1');
711
+ await writer.save({
712
+ sourceTable: tableB,
713
+ tag: storage.SaveOperationTag.INSERT,
714
+ after: firstState,
715
+ afterReplicaId: replicaId
716
+ });
717
+ await writer.commit('1/1');
718
+ let checkpoint = await bucketStorage.getCheckpoint();
719
+ const parameters = new RequestParameters(new JwtPayload({ sub: 'y1' }), {});
720
+ const querier = sync_rules.getBucketParameterQuerier({
721
+ ...test_utils.querierOptions(parameters)
722
+ }).querier;
723
+ let buckets = await querier.queryDynamicBucketDescriptions({
724
+ async getParameterSets(lookups) {
725
+ expect(lookups.map((l) => l.indexKey)).toEqual([['y1']]);
726
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
727
+ expect(parameter_sets).toEqual([{ '0': 'x1' }, { '0': 'x2' }]);
728
+ return parameter_sets;
729
+ }
730
+ });
731
+ expect(buckets).toMatchObject([
732
+ {
733
+ bucket: expect.stringMatching(/chat.*\["x1"\]$/),
734
+ definition: 'chat',
735
+ inclusion_reasons: ['default'],
736
+ priority: 3
737
+ },
738
+ {
739
+ bucket: expect.stringMatching(/chat.*\["x2"\]$/),
740
+ definition: 'chat',
741
+ inclusion_reasons: ['default'],
742
+ priority: 3
743
+ }
744
+ ]);
745
+ // Make the x2 bucket inaccessible
746
+ await writer.save({
747
+ sourceTable: tableB,
748
+ tag: storage.SaveOperationTag.UPDATE,
749
+ before: firstState,
750
+ after: secondState,
751
+ beforeReplicaId: replicaId,
752
+ afterReplicaId: replicaId
753
+ });
754
+ await writer.commit('1/2');
755
+ checkpoint = await bucketStorage.getCheckpoint();
756
+ buckets = await querier.queryDynamicBucketDescriptions({
757
+ async getParameterSets(lookups) {
758
+ expect(lookups.map((l) => l.indexKey)).toEqual([['y1']]);
759
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
760
+ expect(parameter_sets).toEqual([{ '0': 'x2' }]);
761
+ return parameter_sets;
762
+ }
763
+ });
764
+ expect(buckets.map((bkt) => bkt.bucket)).toStrictEqual([expect.stringContaining('x2')]);
765
+ // Second update, remove user from inputs
766
+ await writer.save({
767
+ sourceTable: tableB,
768
+ tag: storage.SaveOperationTag.UPDATE,
769
+ before: secondState,
770
+ after: thirdState,
771
+ beforeReplicaId: replicaId,
772
+ afterReplicaId: replicaId
773
+ });
774
+ await writer.commit('1/3');
775
+ checkpoint = await bucketStorage.getCheckpoint();
776
+ buckets = await querier.queryDynamicBucketDescriptions({
777
+ async getParameterSets(lookups) {
778
+ expect(lookups.map((l) => l.indexKey)).toEqual([['y1']]);
779
+ const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
780
+ expect(parameter_sets).toHaveLength(0);
781
+ return parameter_sets;
782
+ }
783
+ });
784
+ expect(buckets).toHaveLength(0);
755
785
  });
756
786
  }
757
787
  //# sourceMappingURL=register-data-storage-parameter-tests.js.map