@powersync/service-module-mysql 0.0.0-dev-20241015210820 → 0.0.0-dev-20241021185145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -9
- package/dist/replication/BinLogReplicationJob.js +5 -8
- package/dist/replication/BinLogReplicationJob.js.map +1 -1
- package/dist/replication/BinLogStream.d.ts +3 -0
- package/dist/replication/BinLogStream.js +129 -107
- package/dist/replication/BinLogStream.js.map +1 -1
- package/package.json +9 -9
- package/src/replication/BinLogReplicationJob.ts +6 -9
- package/src/replication/BinLogStream.ts +146 -122
- package/src/replication/zongji/zongji.d.ts +15 -0
- package/test/src/binlog_stream.test.ts +191 -192
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -29,10 +29,10 @@ function defineBinlogStreamTests(factory: StorageFactory) {
|
|
|
29
29
|
binlogStreamTest(factory, async (context) => {
|
|
30
30
|
const { connectionManager } = context;
|
|
31
31
|
await context.updateSyncRules(`
|
|
32
|
-
bucket_definitions:
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
bucket_definitions:
|
|
33
|
+
global:
|
|
34
|
+
data:
|
|
35
|
+
- SELECT id, description, num FROM "test_data"`);
|
|
36
36
|
|
|
37
37
|
await connectionManager.query(
|
|
38
38
|
`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description TEXT, num BIGINT)`
|
|
@@ -64,225 +64,224 @@ bucket_definitions:
|
|
|
64
64
|
})
|
|
65
65
|
);
|
|
66
66
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
67
|
+
test(
|
|
68
|
+
'replicating case sensitive table',
|
|
69
|
+
binlogStreamTest(factory, async (context) => {
|
|
70
|
+
const { connectionManager } = context;
|
|
71
|
+
await context.updateSyncRules(`
|
|
72
|
+
bucket_definitions:
|
|
73
|
+
global:
|
|
74
|
+
data:
|
|
75
|
+
- SELECT id, description FROM "test_DATA"
|
|
76
|
+
`);
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
await connectionManager.query(
|
|
79
|
+
`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
|
|
80
|
+
);
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
await context.replicateSnapshot();
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
const startRowCount =
|
|
85
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
|
|
86
|
+
const startTxCount =
|
|
87
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
|
|
88
88
|
|
|
89
|
-
|
|
89
|
+
context.startStreaming();
|
|
90
90
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
await connectionManager.query(`INSERT INTO test_DATA(description) VALUES('test1')`);
|
|
92
|
+
const [[result]] = await connectionManager.query(
|
|
93
|
+
`SELECT id AS test_id FROM test_DATA WHERE description = 'test1'`
|
|
94
|
+
);
|
|
95
|
+
const testId = result.test_id;
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
const data = await context.getBucketData('global[]');
|
|
98
98
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
99
|
+
expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]);
|
|
100
|
+
const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
|
|
101
|
+
const endTxCount =
|
|
102
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
|
|
103
|
+
expect(endRowCount - startRowCount).toEqual(1);
|
|
104
|
+
expect(endTxCount - startTxCount).toEqual(1);
|
|
105
|
+
})
|
|
106
|
+
);
|
|
107
107
|
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
//
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
108
|
+
// TODO: Not supported yet
|
|
109
|
+
// test(
|
|
110
|
+
// 'replicating TRUNCATE',
|
|
111
|
+
// binlogStreamTest(factory, async (context) => {
|
|
112
|
+
// const { connectionManager } = context;
|
|
113
|
+
// const syncRuleContent = `
|
|
114
|
+
// bucket_definitions:
|
|
115
|
+
// global:
|
|
116
|
+
// data:
|
|
117
|
+
// - SELECT id, description FROM "test_data"
|
|
118
|
+
// by_test_data:
|
|
119
|
+
// parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
|
|
120
|
+
// data: []
|
|
121
|
+
// `;
|
|
122
|
+
// await context.updateSyncRules(syncRuleContent);
|
|
123
|
+
// await connectionManager.query(`DROP TABLE IF EXISTS test_data`);
|
|
124
|
+
// await connectionManager.query(
|
|
125
|
+
// `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`
|
|
126
|
+
// );
|
|
127
|
+
//
|
|
128
|
+
// await context.replicateSnapshot();
|
|
129
|
+
// context.startStreaming();
|
|
130
|
+
//
|
|
131
|
+
// const [{ test_id }] = pgwireRows(
|
|
132
|
+
// await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
|
|
133
|
+
// );
|
|
134
|
+
// await connectionManager.query(`TRUNCATE test_data`);
|
|
135
|
+
//
|
|
136
|
+
// const data = await context.getBucketData('global[]');
|
|
137
|
+
//
|
|
138
|
+
// expect(data).toMatchObject([
|
|
139
|
+
// putOp('test_data', { id: test_id, description: 'test1' }),
|
|
140
|
+
// removeOp('test_data', test_id)
|
|
141
|
+
// ]);
|
|
142
|
+
// })
|
|
143
|
+
// );
|
|
144
144
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
145
|
+
test(
|
|
146
|
+
'replicating changing primary key',
|
|
147
|
+
binlogStreamTest(factory, async (context) => {
|
|
148
|
+
const { connectionManager } = context;
|
|
149
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
150
150
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
151
|
+
await connectionManager.query(
|
|
152
|
+
`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
|
|
153
|
+
);
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
155
|
+
await context.replicateSnapshot();
|
|
156
|
+
context.startStreaming();
|
|
157
157
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
158
|
+
await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1')`);
|
|
159
|
+
const [[result1]] = await connectionManager.query(
|
|
160
|
+
`SELECT id AS test_id FROM test_data WHERE description = 'test1'`
|
|
161
|
+
);
|
|
162
|
+
const testId1 = result1.test_id;
|
|
163
163
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
164
|
+
await connectionManager.query(`UPDATE test_data SET id = UUID(), description = 'test2a' WHERE id = '${testId1}'`);
|
|
165
|
+
const [[result2]] = await connectionManager.query(
|
|
166
|
+
`SELECT id AS test_id FROM test_data WHERE description = 'test2a'`
|
|
167
|
+
);
|
|
168
|
+
const testId2 = result2.test_id;
|
|
169
169
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
170
|
+
// This update may fail replicating with:
|
|
171
|
+
// Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
|
|
172
|
+
await connectionManager.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${testId2}'`);
|
|
173
173
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
174
|
+
// Re-use old id again
|
|
175
|
+
await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}', 'test1b')`);
|
|
176
|
+
await connectionManager.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${testId1}'`);
|
|
177
177
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
178
|
+
const data = await context.getBucketData('global[]');
|
|
179
|
+
expect(data).toMatchObject([
|
|
180
|
+
// Initial insert
|
|
181
|
+
putOp('test_data', { id: testId1, description: 'test1' }),
|
|
182
|
+
// Update id, then description
|
|
183
|
+
removeOp('test_data', testId1),
|
|
184
|
+
putOp('test_data', { id: testId2, description: 'test2a' }),
|
|
185
|
+
putOp('test_data', { id: testId2, description: 'test2b' }),
|
|
186
|
+
// Re-use old id
|
|
187
|
+
putOp('test_data', { id: testId1, description: 'test1b' }),
|
|
188
|
+
putOp('test_data', { id: testId1, description: 'test1c' })
|
|
189
|
+
]);
|
|
190
|
+
})
|
|
191
|
+
);
|
|
192
192
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
193
|
+
test(
|
|
194
|
+
'initial sync',
|
|
195
|
+
binlogStreamTest(factory, async (context) => {
|
|
196
|
+
const { connectionManager } = context;
|
|
197
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
198
|
+
|
|
199
|
+
await connectionManager.query(
|
|
200
|
+
`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1')`);
|
|
204
|
+
const [[result]] = await connectionManager.query(
|
|
205
|
+
`SELECT id AS test_id FROM test_data WHERE description = 'test1'`
|
|
206
|
+
);
|
|
207
|
+
const testId = result.test_id;
|
|
198
208
|
|
|
199
|
-
|
|
200
|
-
// `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
|
|
201
|
-
// );
|
|
209
|
+
await context.replicateSnapshot();
|
|
202
210
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
// const testId = result.test_id;
|
|
211
|
+
const data = await context.getBucketData('global[]');
|
|
212
|
+
expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]);
|
|
213
|
+
})
|
|
214
|
+
);
|
|
208
215
|
|
|
216
|
+
// test(
|
|
217
|
+
// 'record too large',
|
|
218
|
+
// binlogStreamTest(factory, async (context) => {
|
|
219
|
+
// await context.updateSyncRules(`bucket_definitions:
|
|
220
|
+
// global:
|
|
221
|
+
// data:
|
|
222
|
+
// - SELECT id, description, other FROM "test_data"`);
|
|
223
|
+
// const { connectionManager } = context;
|
|
224
|
+
//
|
|
225
|
+
// await connectionManager.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
|
|
226
|
+
//
|
|
209
227
|
// await context.replicateSnapshot();
|
|
228
|
+
//
|
|
229
|
+
// // 4MB
|
|
230
|
+
// const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
|
|
231
|
+
// // 18MB
|
|
232
|
+
// const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
|
|
233
|
+
//
|
|
234
|
+
// await connectionManager.query({
|
|
235
|
+
// statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
|
|
236
|
+
// params: [{ type: 'varchar', value: tooLargeDescription }]
|
|
237
|
+
// });
|
|
238
|
+
// await connectionManager.query({
|
|
239
|
+
// statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
|
|
240
|
+
// params: [{ type: 'varchar', value: largeDescription }]
|
|
241
|
+
// });
|
|
242
|
+
//
|
|
210
243
|
// context.startStreaming();
|
|
211
|
-
|
|
244
|
+
//
|
|
212
245
|
// const data = await context.getBucketData('global[]');
|
|
213
|
-
// expect(data).
|
|
246
|
+
// expect(data.length).toEqual(1);
|
|
247
|
+
// const row = JSON.parse(data[0].data as string);
|
|
248
|
+
// delete row.description;
|
|
249
|
+
// expect(row).toEqual({ id: 't1', other: 'foo' });
|
|
250
|
+
// delete data[0].data;
|
|
251
|
+
// expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
|
|
214
252
|
// })
|
|
215
253
|
// );
|
|
216
254
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
// // data:
|
|
223
|
-
// // - SELECT id, description, other FROM "test_data"`);
|
|
224
|
-
// // const { connectionManager } = context;
|
|
225
|
-
// //
|
|
226
|
-
// // await connectionManager.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
|
|
227
|
-
// //
|
|
228
|
-
// // await context.replicateSnapshot();
|
|
229
|
-
// //
|
|
230
|
-
// // // 4MB
|
|
231
|
-
// // const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
|
|
232
|
-
// // // 18MB
|
|
233
|
-
// // const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
|
|
234
|
-
// //
|
|
235
|
-
// // await connectionManager.query({
|
|
236
|
-
// // statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
|
|
237
|
-
// // params: [{ type: 'varchar', value: tooLargeDescription }]
|
|
238
|
-
// // });
|
|
239
|
-
// // await connectionManager.query({
|
|
240
|
-
// // statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
|
|
241
|
-
// // params: [{ type: 'varchar', value: largeDescription }]
|
|
242
|
-
// // });
|
|
243
|
-
// //
|
|
244
|
-
// // context.startStreaming();
|
|
245
|
-
// //
|
|
246
|
-
// // const data = await context.getBucketData('global[]');
|
|
247
|
-
// // expect(data.length).toEqual(1);
|
|
248
|
-
// // const row = JSON.parse(data[0].data as string);
|
|
249
|
-
// // delete row.description;
|
|
250
|
-
// // expect(row).toEqual({ id: 't1', other: 'foo' });
|
|
251
|
-
// // delete data[0].data;
|
|
252
|
-
// // expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
|
|
253
|
-
// // })
|
|
254
|
-
// // );
|
|
255
|
-
|
|
256
|
-
// test(
|
|
257
|
-
// 'table not in sync rules',
|
|
258
|
-
// binlogStreamTest(factory, async (context) => {
|
|
259
|
-
// const { connectionManager } = context;
|
|
260
|
-
// await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
255
|
+
test(
|
|
256
|
+
'table not in sync rules',
|
|
257
|
+
binlogStreamTest(factory, async (context) => {
|
|
258
|
+
const { connectionManager } = context;
|
|
259
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
261
260
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
261
|
+
await connectionManager.query(
|
|
262
|
+
`CREATE TABLE test_donotsync (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
|
|
263
|
+
);
|
|
265
264
|
|
|
266
|
-
|
|
265
|
+
await context.replicateSnapshot();
|
|
267
266
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
267
|
+
const startRowCount =
|
|
268
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
|
|
269
|
+
const startTxCount =
|
|
270
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
|
|
272
271
|
|
|
273
|
-
|
|
272
|
+
context.startStreaming();
|
|
274
273
|
|
|
275
|
-
|
|
276
|
-
|
|
274
|
+
await connectionManager.query(`INSERT INTO test_donotsync(description) VALUES('test1')`);
|
|
275
|
+
const data = await context.getBucketData('global[]');
|
|
277
276
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
277
|
+
expect(data).toMatchObject([]);
|
|
278
|
+
const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
|
|
279
|
+
const endTxCount =
|
|
280
|
+
(await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
|
|
282
281
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
282
|
+
// There was a transaction, but we should not replicate any actual data
|
|
283
|
+
expect(endRowCount - startRowCount).toEqual(0);
|
|
284
|
+
expect(endTxCount - startTxCount).toEqual(1);
|
|
285
|
+
})
|
|
286
|
+
);
|
|
288
287
|
}
|