lakesync 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/adapter-types-DwsQGQS4.d.ts +94 -0
  2. package/dist/adapter.d.ts +202 -63
  3. package/dist/adapter.js +20 -5
  4. package/dist/analyst.js +2 -2
  5. package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Y7ORYgUv.d.ts} +78 -19
  6. package/dist/catalogue.d.ts +1 -1
  7. package/dist/catalogue.js +3 -3
  8. package/dist/{chunk-P3FT7QCW.js → chunk-4SG66H5K.js} +395 -252
  9. package/dist/chunk-4SG66H5K.js.map +1 -0
  10. package/dist/{chunk-GUJWMK5P.js → chunk-C4KD6YKP.js} +419 -380
  11. package/dist/chunk-C4KD6YKP.js.map +1 -0
  12. package/dist/chunk-DGUM43GV.js +11 -0
  13. package/dist/{chunk-IRJ4QRWV.js → chunk-FIIHPQMQ.js} +396 -209
  14. package/dist/chunk-FIIHPQMQ.js.map +1 -0
  15. package/dist/{chunk-UAUQGP3B.js → chunk-U2NV4DUX.js} +2 -2
  16. package/dist/{chunk-NCZYFZ3B.js → chunk-XVP5DJJ7.js} +44 -18
  17. package/dist/{chunk-NCZYFZ3B.js.map → chunk-XVP5DJJ7.js.map} +1 -1
  18. package/dist/{chunk-FHVTUKXL.js → chunk-YHYBLU6W.js} +2 -2
  19. package/dist/{chunk-QMS7TGFL.js → chunk-ZNY4DSFU.js} +29 -15
  20. package/dist/{chunk-QMS7TGFL.js.map → chunk-ZNY4DSFU.js.map} +1 -1
  21. package/dist/{chunk-SF7Y6ZUA.js → chunk-ZU7RC7CT.js} +2 -2
  22. package/dist/client.d.ts +186 -17
  23. package/dist/client.js +456 -188
  24. package/dist/client.js.map +1 -1
  25. package/dist/compactor.d.ts +2 -2
  26. package/dist/compactor.js +4 -4
  27. package/dist/connector-jira.d.ts +13 -3
  28. package/dist/connector-jira.js +7 -3
  29. package/dist/connector-salesforce.d.ts +13 -3
  30. package/dist/connector-salesforce.js +7 -3
  31. package/dist/{coordinator-D32a5rNk.d.ts → coordinator-eGmZMnJ_.d.ts} +120 -30
  32. package/dist/create-poller-Cc2MGfhh.d.ts +55 -0
  33. package/dist/factory-DFfR-030.d.ts +33 -0
  34. package/dist/gateway-server.d.ts +516 -119
  35. package/dist/gateway-server.js +1201 -4035
  36. package/dist/gateway-server.js.map +1 -1
  37. package/dist/gateway.d.ts +69 -106
  38. package/dist/gateway.js +13 -6
  39. package/dist/index.d.ts +65 -58
  40. package/dist/index.js +18 -4
  41. package/dist/parquet.d.ts +1 -1
  42. package/dist/parquet.js +3 -3
  43. package/dist/proto.d.ts +1 -1
  44. package/dist/proto.js +3 -3
  45. package/dist/react.d.ts +47 -10
  46. package/dist/react.js +88 -40
  47. package/dist/react.js.map +1 -1
  48. package/dist/{registry-CPTgO9jv.d.ts → registry-Dd8JuW8T.d.ts} +19 -4
  49. package/dist/{gateway-Bpvatd9n.d.ts → request-handler-B1I5xDOx.d.ts} +193 -20
  50. package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
  51. package/dist/{src-RHKJFQKR.js → src-WU7IBVC4.js} +19 -5
  52. package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
  53. package/dist/{types-D-E0VrfS.d.ts → types-D2C9jTbL.d.ts} +39 -22
  54. package/package.json +1 -1
  55. package/dist/auth-CAVutXzx.d.ts +0 -30
  56. package/dist/chunk-7D4SUZUM.js +0 -38
  57. package/dist/chunk-GUJWMK5P.js.map +0 -1
  58. package/dist/chunk-IRJ4QRWV.js.map +0 -1
  59. package/dist/chunk-P3FT7QCW.js.map +0 -1
  60. package/dist/db-types-BlN-4KbQ.d.ts +0 -29
  61. package/dist/src-CLCALYDT.js +0 -25
  62. package/dist/src-FPJQYQNA.js +0 -27
  63. package/dist/src-FPJQYQNA.js.map +0 -1
  64. package/dist/src-RHKJFQKR.js.map +0 -1
  65. package/dist/types-DSC_EiwR.d.ts +0 -45
  66. /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
  67. /package/dist/{chunk-UAUQGP3B.js.map → chunk-U2NV4DUX.js.map} +0 -0
  68. /package/dist/{chunk-FHVTUKXL.js.map → chunk-YHYBLU6W.js.map} +0 -0
  69. /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-ZU7RC7CT.js.map} +0 -0
  70. /package/dist/{src-CLCALYDT.js.map → src-WU7IBVC4.js.map} +0 -0
@@ -2,8 +2,9 @@ import {
2
2
  AdapterError,
3
3
  Err,
4
4
  Ok,
5
+ isMaterialisable,
5
6
  toError
6
- } from "./chunk-P3FT7QCW.js";
7
+ } from "./chunk-4SG66H5K.js";
7
8
 
8
9
  // ../adapter/src/db-types.ts
9
10
  var BIGQUERY_TYPE_MAP = {
@@ -16,45 +17,30 @@ var BIGQUERY_TYPE_MAP = {
16
17
  function lakeSyncTypeToBigQuery(type) {
17
18
  return BIGQUERY_TYPE_MAP[type];
18
19
  }
19
- function isDatabaseAdapter(adapter) {
20
- return adapter !== null && typeof adapter === "object" && "insertDeltas" in adapter && "queryDeltasSince" in adapter && typeof adapter.insertDeltas === "function";
21
- }
22
20
 
23
- // ../adapter/src/materialise.ts
24
- function isMaterialisable(adapter) {
25
- return adapter !== null && typeof adapter === "object" && "materialise" in adapter && typeof adapter.materialise === "function";
26
- }
27
- function resolvePrimaryKey(schema) {
28
- return schema.primaryKey ?? ["row_id"];
29
- }
30
- function resolveConflictColumns(schema) {
31
- return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
32
- }
33
- function isSoftDelete(schema) {
34
- return schema.softDelete !== false;
35
- }
36
- function groupDeltasByTable(deltas) {
37
- const result = /* @__PURE__ */ new Map();
38
- for (const delta of deltas) {
39
- let rowIds = result.get(delta.table);
40
- if (!rowIds) {
41
- rowIds = /* @__PURE__ */ new Set();
42
- result.set(delta.table, rowIds);
21
+ // ../adapter/src/shared.ts
22
+ function groupAndMerge(rows) {
23
+ const byRowId = /* @__PURE__ */ new Map();
24
+ for (const row of rows) {
25
+ let arr = byRowId.get(row.row_id);
26
+ if (!arr) {
27
+ arr = [];
28
+ byRowId.set(row.row_id, arr);
43
29
  }
44
- rowIds.add(delta.rowId);
30
+ arr.push(row);
45
31
  }
46
- return result;
47
- }
48
- function buildSchemaIndex(schemas) {
49
- const index = /* @__PURE__ */ new Map();
50
- for (const schema of schemas) {
51
- const key = schema.sourceTable ?? schema.table;
52
- index.set(key, schema);
32
+ const upserts = [];
33
+ const deleteIds = [];
34
+ for (const [rowId, group] of byRowId) {
35
+ const state = mergeLatestState(group);
36
+ if (state !== null) {
37
+ upserts.push({ rowId, state });
38
+ } else {
39
+ deleteIds.push(rowId);
40
+ }
53
41
  }
54
- return index;
42
+ return { upserts, deleteIds };
55
43
  }
56
-
57
- // ../adapter/src/shared.ts
58
44
  function toCause(error) {
59
45
  return error instanceof Error ? error : void 0;
60
46
  }
@@ -89,6 +75,69 @@ function mergeLatestState(rows) {
89
75
  return state;
90
76
  }
91
77
 
78
+ // ../adapter/src/materialise.ts
79
+ function resolvePrimaryKey(schema) {
80
+ return schema.primaryKey ?? ["row_id"];
81
+ }
82
+ function resolveConflictColumns(schema) {
83
+ return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
84
+ }
85
+ function isSoftDelete(schema) {
86
+ return schema.softDelete !== false;
87
+ }
88
+ function groupDeltasByTable(deltas) {
89
+ const result = /* @__PURE__ */ new Map();
90
+ for (const delta of deltas) {
91
+ let rowIds = result.get(delta.table);
92
+ if (!rowIds) {
93
+ rowIds = /* @__PURE__ */ new Set();
94
+ result.set(delta.table, rowIds);
95
+ }
96
+ rowIds.add(delta.rowId);
97
+ }
98
+ return result;
99
+ }
100
+ function buildSchemaIndex(schemas) {
101
+ const index = /* @__PURE__ */ new Map();
102
+ for (const schema of schemas) {
103
+ const key = schema.sourceTable ?? schema.table;
104
+ index.set(key, schema);
105
+ }
106
+ return index;
107
+ }
108
+ async function executeMaterialise(executor, dialect, deltas, schemas) {
109
+ if (deltas.length === 0) {
110
+ return Ok(void 0);
111
+ }
112
+ return wrapAsync(async () => {
113
+ const grouped = groupDeltasByTable(deltas);
114
+ const schemaIndex = buildSchemaIndex(schemas);
115
+ for (const [tableName, rowIds] of grouped) {
116
+ const schema = schemaIndex.get(tableName);
117
+ if (!schema) continue;
118
+ const dest = schema.table;
119
+ const pk = resolvePrimaryKey(schema);
120
+ const conflictCols = resolveConflictColumns(schema);
121
+ const soft = isSoftDelete(schema);
122
+ const createStmt = dialect.createDestinationTable(dest, schema, pk, soft);
123
+ await executor.query(createStmt.sql, createStmt.params);
124
+ const sourceTable = schema.sourceTable ?? schema.table;
125
+ const rowIdArray = [...rowIds];
126
+ const historyStmt = dialect.queryDeltaHistory(sourceTable, rowIdArray);
127
+ const rows = await executor.queryRows(historyStmt.sql, historyStmt.params);
128
+ const { upserts, deleteIds } = groupAndMerge(rows);
129
+ if (upserts.length > 0) {
130
+ const upsertStmt = dialect.buildUpsert(dest, schema, conflictCols, soft, upserts);
131
+ await executor.query(upsertStmt.sql, upsertStmt.params);
132
+ }
133
+ if (deleteIds.length > 0) {
134
+ const deleteStmt = dialect.buildDelete(dest, deleteIds, soft);
135
+ await executor.query(deleteStmt.sql, deleteStmt.params);
136
+ }
137
+ }
138
+ }, "Failed to materialise deltas");
139
+ }
140
+
92
141
  // ../adapter/src/bigquery.ts
93
142
  import { BigQuery } from "@google-cloud/bigquery";
94
143
  function rowToRowDelta(row) {
@@ -105,6 +154,121 @@ function rowToRowDelta(row) {
105
154
  op: row.op
106
155
  };
107
156
  }
157
+ var BigQuerySqlDialect = class {
158
+ constructor(dataset) {
159
+ this.dataset = dataset;
160
+ }
161
+ createDestinationTable(dest, schema, pk, softDelete) {
162
+ const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
163
+ const deletedAtCol = softDelete ? `,
164
+ deleted_at TIMESTAMP` : "";
165
+ return {
166
+ sql: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${dest}\` (
167
+ row_id STRING NOT NULL,
168
+ ${colDefs},
169
+ props JSON DEFAULT '{}'${deletedAtCol},
170
+ synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
171
+ )
172
+ CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
173
+ params: []
174
+ };
175
+ }
176
+ queryDeltaHistory(sourceTable, rowIds) {
177
+ return {
178
+ sql: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
179
+ WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
180
+ ORDER BY hlc ASC`,
181
+ params: [
182
+ ["sourceTable", sourceTable],
183
+ ["rowIds", rowIds]
184
+ ]
185
+ };
186
+ }
187
+ buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
188
+ const namedParams = [];
189
+ const selects = [];
190
+ for (let i = 0; i < upserts.length; i++) {
191
+ const u = upserts[i];
192
+ namedParams.push([`rid_${i}`, u.rowId]);
193
+ for (const col of schema.columns) {
194
+ namedParams.push([`c${schema.columns.indexOf(col)}_${i}`, u.state[col.name] ?? null]);
195
+ }
196
+ const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
197
+ const deletedAtSelect = softDelete ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
198
+ selects.push(
199
+ `SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
200
+ );
201
+ }
202
+ const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
203
+ const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
204
+ const softUpdateExtra = softDelete ? ", deleted_at = s.deleted_at" : "";
205
+ const insertColsList = [
206
+ "row_id",
207
+ ...schema.columns.map((c) => c.name),
208
+ "props",
209
+ ...softDelete ? ["deleted_at"] : [],
210
+ "synced_at"
211
+ ].join(", ");
212
+ const insertValsList = [
213
+ "s.row_id",
214
+ ...schema.columns.map((c) => `s.${c.name}`),
215
+ "'{}'",
216
+ ...softDelete ? ["s.deleted_at"] : [],
217
+ "s.synced_at"
218
+ ].join(", ");
219
+ return {
220
+ sql: `MERGE \`${this.dataset}.${dest}\` AS t
221
+ USING (${selects.join(" UNION ALL ")}) AS s
222
+ ON ${mergeOn}
223
+ WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
224
+ WHEN NOT MATCHED THEN INSERT (${insertColsList})
225
+ VALUES (${insertValsList})`,
226
+ params: namedParams
227
+ };
228
+ }
229
+ buildDelete(dest, deleteIds, softDelete) {
230
+ if (softDelete) {
231
+ return {
232
+ sql: `UPDATE \`${this.dataset}.${dest}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
233
+ params: [["rowIds", deleteIds]]
234
+ };
235
+ }
236
+ return {
237
+ sql: `DELETE FROM \`${this.dataset}.${dest}\` WHERE row_id IN UNNEST(@rowIds)`,
238
+ params: [["rowIds", deleteIds]]
239
+ };
240
+ }
241
+ };
242
+ function createBigQueryExecutor(client, location) {
243
+ function toNamedParams(params) {
244
+ if (params.length === 0) return void 0;
245
+ const result = {};
246
+ for (const entry of params) {
247
+ const [key, value] = entry;
248
+ result[key] = value;
249
+ }
250
+ return result;
251
+ }
252
+ return {
253
+ async query(sql, params) {
254
+ const namedParams = toNamedParams(params);
255
+ await client.query({
256
+ query: sql,
257
+ params: namedParams,
258
+ location
259
+ });
260
+ },
261
+ async queryRows(sql, params) {
262
+ const namedParams = toNamedParams(params);
263
+ const [rows] = await client.query({
264
+ query: sql,
265
+ params: namedParams,
266
+ location
267
+ });
268
+ return rows;
269
+ }
270
+ };
271
+ }
108
272
  var BigQueryAdapter = class {
109
273
  /** @internal */
110
274
  client;
@@ -112,6 +276,8 @@ var BigQueryAdapter = class {
112
276
  dataset;
113
277
  /** @internal */
114
278
  location;
279
+ dialect;
280
+ executor;
115
281
  constructor(config) {
116
282
  this.client = new BigQuery({
117
283
  projectId: config.projectId,
@@ -119,6 +285,8 @@ var BigQueryAdapter = class {
119
285
  });
120
286
  this.dataset = config.dataset;
121
287
  this.location = config.location ?? "US";
288
+ this.dialect = new BigQuerySqlDialect(this.dataset);
289
+ this.executor = createBigQueryExecutor(this.client, this.location);
122
290
  }
123
291
  /**
124
292
  * Insert deltas into the database in a single batch.
@@ -231,125 +399,11 @@ CLUSTER BY \`table\`, hlc`,
231
399
  /**
232
400
  * Materialise deltas into destination tables.
233
401
  *
234
- * For each affected table, queries the full delta history for touched rows,
235
- * merges to latest state via column-level LWW, then upserts live rows and
236
- * deletes tombstoned rows. The consumer-owned `props` column is never
237
- * touched on UPDATE.
402
+ * Delegates to the shared `executeMaterialise` algorithm with the
403
+ * BigQuery SQL dialect.
238
404
  */
239
405
  async materialise(deltas, schemas) {
240
- if (deltas.length === 0) {
241
- return Ok(void 0);
242
- }
243
- return wrapAsync(async () => {
244
- const tableRowIds = groupDeltasByTable(deltas);
245
- const schemaIndex = buildSchemaIndex(schemas);
246
- for (const [sourceTable, rowIds] of tableRowIds) {
247
- const schema = schemaIndex.get(sourceTable);
248
- if (!schema) continue;
249
- const pk = resolvePrimaryKey(schema);
250
- const conflictCols = resolveConflictColumns(schema);
251
- const soft = isSoftDelete(schema);
252
- const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
253
- const deletedAtCol = soft ? `,
254
- deleted_at TIMESTAMP` : "";
255
- await this.client.query({
256
- query: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${schema.table}\` (
257
- row_id STRING NOT NULL,
258
- ${colDefs},
259
- props JSON DEFAULT '{}'${deletedAtCol},
260
- synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
261
- )
262
- CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
263
- location: this.location
264
- });
265
- const rowIdArray = [...rowIds];
266
- const [deltaRows] = await this.client.query({
267
- query: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
268
- WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
269
- ORDER BY hlc ASC`,
270
- params: { sourceTable, rowIds: rowIdArray },
271
- location: this.location
272
- });
273
- const rowGroups = /* @__PURE__ */ new Map();
274
- for (const row of deltaRows) {
275
- let group = rowGroups.get(row.row_id);
276
- if (!group) {
277
- group = [];
278
- rowGroups.set(row.row_id, group);
279
- }
280
- group.push({ columns: row.columns, op: row.op });
281
- }
282
- const upserts = [];
283
- const deleteRowIds = [];
284
- for (const [rowId, group] of rowGroups) {
285
- const state = mergeLatestState(group);
286
- if (state === null) {
287
- deleteRowIds.push(rowId);
288
- } else {
289
- upserts.push({ rowId, state });
290
- }
291
- }
292
- if (upserts.length > 0) {
293
- const params = {};
294
- const selects = [];
295
- for (let i = 0; i < upserts.length; i++) {
296
- const u = upserts[i];
297
- params[`rid_${i}`] = u.rowId;
298
- for (const col of schema.columns) {
299
- params[`c${schema.columns.indexOf(col)}_${i}`] = u.state[col.name] ?? null;
300
- }
301
- const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
302
- const deletedAtSelect = soft ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
303
- selects.push(
304
- `SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
305
- );
306
- }
307
- const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
308
- const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
309
- const softUpdateExtra = soft ? ", deleted_at = s.deleted_at" : "";
310
- const insertColsList = [
311
- "row_id",
312
- ...schema.columns.map((c) => c.name),
313
- "props",
314
- ...soft ? ["deleted_at"] : [],
315
- "synced_at"
316
- ].join(", ");
317
- const insertValsList = [
318
- "s.row_id",
319
- ...schema.columns.map((c) => `s.${c.name}`),
320
- "'{}'",
321
- ...soft ? ["s.deleted_at"] : [],
322
- "s.synced_at"
323
- ].join(", ");
324
- const mergeSql = `MERGE \`${this.dataset}.${schema.table}\` AS t
325
- USING (${selects.join(" UNION ALL ")}) AS s
326
- ON ${mergeOn}
327
- WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
328
- WHEN NOT MATCHED THEN INSERT (${insertColsList})
329
- VALUES (${insertValsList})`;
330
- await this.client.query({
331
- query: mergeSql,
332
- params,
333
- location: this.location
334
- });
335
- }
336
- if (deleteRowIds.length > 0) {
337
- if (soft) {
338
- await this.client.query({
339
- query: `UPDATE \`${this.dataset}.${schema.table}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
340
- params: { rowIds: deleteRowIds },
341
- location: this.location
342
- });
343
- } else {
344
- await this.client.query({
345
- query: `DELETE FROM \`${this.dataset}.${schema.table}\` WHERE row_id IN UNNEST(@rowIds)`,
346
- params: { rowIds: deleteRowIds },
347
- location: this.location
348
- });
349
- }
350
- }
351
- }
352
- }, "Failed to materialise deltas");
406
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
353
407
  }
354
408
  /**
355
409
  * No-op — BigQuery client is HTTP-based with no persistent connections.
@@ -460,11 +514,79 @@ var MYSQL_TYPE_MAP = {
460
514
  function lakeSyncTypeToMySQL(type) {
461
515
  return MYSQL_TYPE_MAP[type];
462
516
  }
517
+ var MySqlDialect = class {
518
+ createDestinationTable(dest, schema, pk, softDelete) {
519
+ const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
520
+ const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
521
+ const deletedAtCol = softDelete ? `, deleted_at TIMESTAMP NULL` : "";
522
+ const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
523
+ return {
524
+ sql: `CREATE TABLE IF NOT EXISTS \`${dest}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`,
525
+ params: []
526
+ };
527
+ }
528
+ queryDeltaHistory(sourceTable, rowIds) {
529
+ const placeholders = rowIds.map(() => "?").join(", ");
530
+ return {
531
+ sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
532
+ params: [sourceTable, ...rowIds]
533
+ };
534
+ }
535
+ buildUpsert(dest, schema, _conflictCols, softDelete, upserts) {
536
+ const cols = schema.columns.map((c) => c.name);
537
+ const valuePlaceholders = softDelete ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
538
+ const values = [];
539
+ for (const { rowId, state } of upserts) {
540
+ values.push(rowId);
541
+ for (const col of cols) {
542
+ values.push(state[col] ?? null);
543
+ }
544
+ }
545
+ const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
546
+ const softUpdateExtra = softDelete ? ", deleted_at = NULL" : "";
547
+ const colList = softDelete ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
548
+ return {
549
+ sql: `INSERT INTO \`${dest}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
550
+ params: values
551
+ };
552
+ }
553
+ buildDelete(dest, deleteIds, softDelete) {
554
+ const placeholders = deleteIds.map(() => "?").join(", ");
555
+ if (softDelete) {
556
+ return {
557
+ sql: `UPDATE \`${dest}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${placeholders})`,
558
+ params: deleteIds
559
+ };
560
+ }
561
+ return {
562
+ sql: `DELETE FROM \`${dest}\` WHERE row_id IN (${placeholders})`,
563
+ params: deleteIds
564
+ };
565
+ }
566
+ };
463
567
  var MySQLAdapter = class {
464
568
  /** @internal */
465
569
  pool;
570
+ dialect = new MySqlDialect();
466
571
  constructor(config) {
467
- this.pool = mysql.createPool(config.connectionString);
572
+ this.pool = mysql.createPool({
573
+ uri: config.connectionString,
574
+ connectionLimit: config.poolMax ?? 10,
575
+ connectTimeout: config.connectionTimeoutMs ?? 3e4,
576
+ idleTimeout: config.idleTimeoutMs ?? 1e4
577
+ });
578
+ }
579
+ get executor() {
580
+ const pool = this.pool;
581
+ return {
582
+ async query(sql, params) {
583
+ await pool.execute(sql, params);
584
+ },
585
+ async queryRows(sql, params) {
586
+ const [rows] = await pool.execute(sql, params);
587
+ return rows;
588
+ }
589
+ };
468
590
  }
469
591
  /**
470
592
  * Insert deltas into the database in a single batch.
@@ -547,89 +669,11 @@ var MySQLAdapter = class {
547
669
  /**
548
670
  * Materialise deltas into destination tables.
549
671
  *
550
- * For each table with a matching schema, merges delta history into the
551
- * latest row state and upserts into the destination table. Tombstoned
552
- * rows are soft-deleted (default) or hard-deleted. The `props` column
553
- * is never touched.
672
+ * Delegates to the shared `executeMaterialise` algorithm with the
673
+ * MySQL SQL dialect.
554
674
  */
555
675
  async materialise(deltas, schemas) {
556
- if (deltas.length === 0) {
557
- return Ok(void 0);
558
- }
559
- return wrapAsync(async () => {
560
- const grouped = groupDeltasByTable(deltas);
561
- const schemaIndex = buildSchemaIndex(schemas);
562
- for (const [tableName, rowIds] of grouped) {
563
- const schema = schemaIndex.get(tableName);
564
- if (!schema) continue;
565
- const pk = resolvePrimaryKey(schema);
566
- const soft = isSoftDelete(schema);
567
- const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
568
- const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
569
- const deletedAtCol = soft ? `, deleted_at TIMESTAMP NULL` : "";
570
- const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
571
- await this.pool.execute(
572
- `CREATE TABLE IF NOT EXISTS \`${schema.table}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`
573
- );
574
- const rowIdArray = [...rowIds];
575
- const placeholders = rowIdArray.map(() => "?").join(", ");
576
- const [rows] = await this.pool.execute(
577
- `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
578
- [tableName, ...rowIdArray]
579
- );
580
- const byRow = /* @__PURE__ */ new Map();
581
- for (const row of rows) {
582
- let list = byRow.get(row.row_id);
583
- if (!list) {
584
- list = [];
585
- byRow.set(row.row_id, list);
586
- }
587
- list.push(row);
588
- }
589
- const upserts = [];
590
- const deleteIds = [];
591
- for (const [rowId, rowDeltas] of byRow) {
592
- const state = mergeLatestState(rowDeltas);
593
- if (state === null) {
594
- deleteIds.push(rowId);
595
- } else {
596
- upserts.push({ rowId, state });
597
- }
598
- }
599
- if (upserts.length > 0) {
600
- const cols = schema.columns.map((c) => c.name);
601
- const valuePlaceholders = soft ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
602
- const values = [];
603
- for (const { rowId, state } of upserts) {
604
- values.push(rowId);
605
- for (const col of cols) {
606
- values.push(state[col] ?? null);
607
- }
608
- }
609
- const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
610
- const softUpdateExtra = soft ? ", deleted_at = NULL" : "";
611
- const colList = soft ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
612
- await this.pool.execute(
613
- `INSERT INTO \`${schema.table}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
614
- values
615
- );
616
- }
617
- if (deleteIds.length > 0) {
618
- const delPlaceholders = deleteIds.map(() => "?").join(", ");
619
- if (soft) {
620
- await this.pool.execute(
621
- `UPDATE \`${schema.table}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${delPlaceholders})`,
622
- deleteIds
623
- );
624
- } else {
625
- await this.pool.execute(
626
- `DELETE FROM \`${schema.table}\` WHERE row_id IN (${delPlaceholders})`,
627
- deleteIds
628
- );
629
- }
630
- }
631
- }
632
- }, "Failed to materialise deltas");
676
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
633
677
  }
634
678
  /** Close the database connection pool and release resources. */
635
679
  async close() {
@@ -657,15 +701,98 @@ var POSTGRES_TYPE_MAP = {
657
701
  json: "JSONB",
658
702
  null: "TEXT"
659
703
  };
704
+ var PostgresSqlDialect = class {
705
+ createDestinationTable(dest, schema, pk, softDelete) {
706
+ const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
707
+ const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
708
+ const deletedAtCol = softDelete ? `,
709
+ deleted_at TIMESTAMPTZ` : "";
710
+ const uniqueConstraint = schema.externalIdColumn ? `,
711
+ UNIQUE ("${schema.externalIdColumn}")` : "";
712
+ return {
713
+ sql: `CREATE TABLE IF NOT EXISTS "${dest}" (
714
+ row_id TEXT NOT NULL,
715
+ ${columnDefs},
716
+ props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
717
+ synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
718
+ ${pkConstraint}${uniqueConstraint}
719
+ )`,
720
+ params: []
721
+ };
722
+ }
723
+ queryDeltaHistory(sourceTable, rowIds) {
724
+ return {
725
+ sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
726
+ params: [sourceTable, rowIds]
727
+ };
728
+ }
729
+ buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
730
+ const colNames = schema.columns.map((c) => c.name);
731
+ const baseCols = ["row_id", ...colNames];
732
+ const allCols = softDelete ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
733
+ const colList = allCols.map((c) => `"${c}"`).join(", ");
734
+ const values = [];
735
+ const valueRows = [];
736
+ const paramsPerRow = allCols.length;
737
+ for (let i = 0; i < upserts.length; i++) {
738
+ const u = upserts[i];
739
+ const offset = i * paramsPerRow;
740
+ const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
741
+ valueRows.push(`(${placeholders.join(", ")})`);
742
+ values.push(u.rowId);
743
+ for (const col of colNames) {
744
+ values.push(u.state[col] ?? null);
745
+ }
746
+ if (softDelete) values.push(null);
747
+ values.push(/* @__PURE__ */ new Date());
748
+ }
749
+ const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
750
+ const updateCols = softDelete ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
751
+ const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
752
+ return {
753
+ sql: `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
754
+ params: values
755
+ };
756
+ }
757
+ buildDelete(dest, deleteIds, softDelete) {
758
+ if (softDelete) {
759
+ return {
760
+ sql: `UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
761
+ params: [deleteIds]
762
+ };
763
+ }
764
+ return {
765
+ sql: `DELETE FROM "${dest}" WHERE row_id = ANY($1)`,
766
+ params: [deleteIds]
767
+ };
768
+ }
769
+ };
660
770
  var PostgresAdapter = class {
661
771
  /** @internal */
662
772
  pool;
773
+ dialect = new PostgresSqlDialect();
663
774
  constructor(config) {
664
775
  const poolConfig = {
665
- connectionString: config.connectionString
776
+ connectionString: config.connectionString,
777
+ max: config.poolMax ?? 10,
778
+ idleTimeoutMillis: config.idleTimeoutMs ?? 1e4,
779
+ connectionTimeoutMillis: config.connectionTimeoutMs ?? 3e4,
780
+ statement_timeout: config.statementTimeoutMs ?? 3e4
666
781
  };
667
782
  this.pool = new Pool(poolConfig);
668
783
  }
784
+ get executor() {
785
+ const pool = this.pool;
786
+ return {
787
+ async query(sql, params) {
788
+ await pool.query(sql, params);
789
+ },
790
+ async queryRows(sql, params) {
791
+ const result = await pool.query(sql, params);
792
+ return result.rows;
793
+ }
794
+ };
795
+ }
669
796
  /**
670
797
  * Insert deltas into the database in a single batch.
671
798
  * Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
@@ -762,105 +889,11 @@ CREATE INDEX IF NOT EXISTS idx_lakesync_deltas_table_row ON lakesync_deltas ("ta
762
889
  /**
763
890
  * Materialise deltas into destination tables.
764
891
  *
765
- * For each table with a matching schema, merges delta history into the
766
- * latest row state and upserts into the destination table. Tombstoned
767
- * rows are deleted. The `props` column is never touched.
892
+ * Delegates to the shared `executeMaterialise` algorithm with the
893
+ * Postgres SQL dialect.
768
894
  */
769
895
  async materialise(deltas, schemas) {
770
- if (deltas.length === 0) {
771
- return Ok(void 0);
772
- }
773
- return wrapAsync(async () => {
774
- const grouped = groupDeltasByTable(deltas);
775
- const schemaIndex = buildSchemaIndex(schemas);
776
- for (const [tableName, rowIds] of grouped) {
777
- const schema = schemaIndex.get(tableName);
778
- if (!schema) continue;
779
- const dest = schema.table;
780
- const pk = resolvePrimaryKey(schema);
781
- const conflictCols = resolveConflictColumns(schema);
782
- const soft = isSoftDelete(schema);
783
- const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
784
- const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
785
- const deletedAtCol = soft ? `,
786
- deleted_at TIMESTAMPTZ` : "";
787
- const uniqueConstraint = schema.externalIdColumn ? `,
788
- UNIQUE ("${schema.externalIdColumn}")` : "";
789
- await this.pool.query(
790
- `CREATE TABLE IF NOT EXISTS "${dest}" (
791
- row_id TEXT NOT NULL,
792
- ${columnDefs},
793
- props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
794
- synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
795
- ${pkConstraint}${uniqueConstraint}
796
- )`
797
- );
798
- const sourceTable = schema.sourceTable ?? schema.table;
799
- const rowIdArray = [...rowIds];
800
- const deltaResult = await this.pool.query(
801
- `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
802
- [sourceTable, rowIdArray]
803
- );
804
- const byRowId = /* @__PURE__ */ new Map();
805
- for (const row of deltaResult.rows) {
806
- const rid = row.row_id;
807
- let arr = byRowId.get(rid);
808
- if (!arr) {
809
- arr = [];
810
- byRowId.set(rid, arr);
811
- }
812
- arr.push(row);
813
- }
814
- const upserts = [];
815
- const deleteIds = [];
816
- for (const [rowId, rows] of byRowId) {
817
- const state = mergeLatestState(rows);
818
- if (state !== null) {
819
- upserts.push({ rowId, state });
820
- } else {
821
- deleteIds.push(rowId);
822
- }
823
- }
824
- if (upserts.length > 0) {
825
- const colNames = schema.columns.map((c) => c.name);
826
- const baseCols = ["row_id", ...colNames];
827
- const allCols = soft ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
828
- const colList = allCols.map((c) => `"${c}"`).join(", ");
829
- const values = [];
830
- const valueRows = [];
831
- const paramsPerRow = allCols.length;
832
- for (let i = 0; i < upserts.length; i++) {
833
- const u = upserts[i];
834
- const offset = i * paramsPerRow;
835
- const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
836
- valueRows.push(`(${placeholders.join(", ")})`);
837
- values.push(u.rowId);
838
- for (const col of colNames) {
839
- values.push(u.state[col] ?? null);
840
- }
841
- if (soft) values.push(null);
842
- values.push(/* @__PURE__ */ new Date());
843
- }
844
- const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
845
- const updateCols = soft ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
846
- const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
847
- await this.pool.query(
848
- `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
849
- values
850
- );
851
- }
852
- if (deleteIds.length > 0) {
853
- if (soft) {
854
- await this.pool.query(
855
- `UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
856
- [deleteIds]
857
- );
858
- } else {
859
- await this.pool.query(`DELETE FROM "${dest}" WHERE row_id = ANY($1)`, [deleteIds]);
860
- }
861
- }
862
- }
863
- }, "Failed to materialise deltas");
896
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
864
897
  }
865
898
  /** Close the database connection pool and release resources. */
866
899
  async close() {
@@ -881,45 +914,46 @@ function rowToRowDelta2(row) {
881
914
  }
882
915
 
883
916
  // ../adapter/src/factory.ts
884
- function createDatabaseAdapter(config) {
885
- try {
886
- switch (config.type) {
887
- case "postgres": {
888
- if (!config.postgres) {
889
- return Err(new AdapterError("Postgres connector config missing postgres field"));
890
- }
891
- return Ok(
892
- new PostgresAdapter({
893
- connectionString: config.postgres.connectionString
894
- })
895
- );
896
- }
897
- case "mysql": {
898
- if (!config.mysql) {
899
- return Err(new AdapterError("MySQL connector config missing mysql field"));
900
- }
901
- return Ok(
902
- new MySQLAdapter({
903
- connectionString: config.mysql.connectionString
904
- })
905
- );
906
- }
907
- case "bigquery": {
908
- if (!config.bigquery) {
909
- return Err(new AdapterError("BigQuery connector config missing bigquery field"));
910
- }
911
- return Ok(
912
- new BigQueryAdapter({
913
- projectId: config.bigquery.projectId,
914
- dataset: config.bigquery.dataset,
915
- keyFilename: config.bigquery.keyFilename,
916
- location: config.bigquery.location
917
- })
918
- );
919
- }
920
- default:
921
- return Err(new AdapterError(`Unsupported connector type: ${config.type}`));
917
+ function createAdapterFactoryRegistry(factories = /* @__PURE__ */ new Map()) {
918
+ return buildAdapterFactoryRegistry(new Map(factories));
919
+ }
920
+ function buildAdapterFactoryRegistry(map) {
921
+ return {
922
+ get(type) {
923
+ return map.get(type);
924
+ },
925
+ with(type, factory) {
926
+ const next = new Map(map);
927
+ next.set(type, factory);
928
+ return buildAdapterFactoryRegistry(next);
922
929
  }
930
+ };
931
+ }
932
+ function defaultAdapterFactoryRegistry() {
933
+ return createAdapterFactoryRegistry().with("postgres", (c) => {
934
+ const pg = c.postgres;
935
+ return new PostgresAdapter({ connectionString: pg.connectionString });
936
+ }).with("mysql", (c) => {
937
+ const my = c.mysql;
938
+ return new MySQLAdapter({ connectionString: my.connectionString });
939
+ }).with("bigquery", (c) => {
940
+ const bq = c.bigquery;
941
+ return new BigQueryAdapter({
942
+ projectId: bq.projectId,
943
+ dataset: bq.dataset,
944
+ keyFilename: bq.keyFilename,
945
+ location: bq.location
946
+ });
947
+ });
948
+ }
949
+ function createDatabaseAdapter(config, registry) {
950
+ const reg = registry ?? defaultAdapterFactoryRegistry();
951
+ const factory = reg.get(config.type);
952
+ if (!factory) {
953
+ return Err(new AdapterError(`No adapter factory for connector type "${config.type}"`));
954
+ }
955
+ try {
956
+ return Ok(factory(config));
923
957
  } catch (err) {
924
958
  return Err(new AdapterError(`Failed to create adapter: ${toError(err).message}`));
925
959
  }
@@ -1214,18 +1248,18 @@ var MinIOAdapter = class {
1214
1248
  async function createQueryFn(config) {
1215
1249
  switch (config.type) {
1216
1250
  case "postgres": {
1217
- if (!config.postgres) return null;
1251
+ const pg = config;
1218
1252
  const { Pool: Pool2 } = await import("pg");
1219
- const pool = new Pool2({ connectionString: config.postgres.connectionString });
1253
+ const pool = new Pool2({ connectionString: pg.postgres.connectionString });
1220
1254
  return async (sql, params) => {
1221
1255
  const result = await pool.query(sql, params);
1222
1256
  return result.rows;
1223
1257
  };
1224
1258
  }
1225
1259
  case "mysql": {
1226
- if (!config.mysql) return null;
1260
+ const my = config;
1227
1261
  const mysql2 = await import("mysql2/promise");
1228
- const pool = mysql2.createPool(config.mysql.connectionString);
1262
+ const pool = mysql2.createPool(my.mysql.connectionString);
1229
1263
  return async (sql, params) => {
1230
1264
  const [rows] = await pool.query(sql, params);
1231
1265
  return rows;
@@ -1238,20 +1272,25 @@ async function createQueryFn(config) {
1238
1272
 
1239
1273
  export {
1240
1274
  lakeSyncTypeToBigQuery,
1241
- isDatabaseAdapter,
1242
- isMaterialisable,
1275
+ groupAndMerge,
1276
+ toCause,
1277
+ wrapAsync,
1278
+ mergeLatestState,
1243
1279
  resolvePrimaryKey,
1244
1280
  resolveConflictColumns,
1245
1281
  isSoftDelete,
1246
1282
  groupDeltasByTable,
1247
1283
  buildSchemaIndex,
1248
- toCause,
1249
- wrapAsync,
1250
- mergeLatestState,
1284
+ executeMaterialise,
1285
+ BigQuerySqlDialect,
1251
1286
  BigQueryAdapter,
1252
1287
  CompositeAdapter,
1288
+ MySqlDialect,
1253
1289
  MySQLAdapter,
1290
+ PostgresSqlDialect,
1254
1291
  PostgresAdapter,
1292
+ createAdapterFactoryRegistry,
1293
+ defaultAdapterFactoryRegistry,
1255
1294
  createDatabaseAdapter,
1256
1295
  FanOutAdapter,
1257
1296
  LifecycleAdapter,
@@ -1260,4 +1299,4 @@ export {
1260
1299
  MinIOAdapter,
1261
1300
  createQueryFn
1262
1301
  };
1263
- //# sourceMappingURL=chunk-GUJWMK5P.js.map
1302
+ //# sourceMappingURL=chunk-C4KD6YKP.js.map