lakesync 0.1.5 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/adapter.d.ts +199 -19
  2. package/dist/adapter.js +19 -3
  3. package/dist/analyst.js +2 -2
  4. package/dist/{base-poller-CBvhdvcj.d.ts → base-poller-Bj9kX9dv.d.ts} +76 -19
  5. package/dist/catalogue.d.ts +1 -1
  6. package/dist/catalogue.js +3 -3
  7. package/dist/chunk-DGUM43GV.js +11 -0
  8. package/dist/{chunk-PWGQ3PXE.js → chunk-JI4C4R5H.js} +280 -140
  9. package/dist/chunk-JI4C4R5H.js.map +1 -0
  10. package/dist/{chunk-L4ZL5JA7.js → chunk-KVSWLIJR.js} +2 -2
  11. package/dist/{chunk-7UBS6MFH.js → chunk-LDFFCG2K.js} +377 -247
  12. package/dist/chunk-LDFFCG2K.js.map +1 -0
  13. package/dist/{chunk-Z7FGLEQU.js → chunk-LPWXOYNS.js} +376 -287
  14. package/dist/chunk-LPWXOYNS.js.map +1 -0
  15. package/dist/{chunk-SZSGSTVZ.js → chunk-PYRS74YP.js} +15 -4
  16. package/dist/{chunk-SZSGSTVZ.js.map → chunk-PYRS74YP.js.map} +1 -1
  17. package/dist/{chunk-TVLTXHW6.js → chunk-QNITY4F6.js} +30 -7
  18. package/dist/{chunk-TVLTXHW6.js.map → chunk-QNITY4F6.js.map} +1 -1
  19. package/dist/{chunk-46CKACNC.js → chunk-SSICS5KI.js} +2 -2
  20. package/dist/{chunk-B3QEUG6E.js → chunk-TMLG32QV.js} +2 -2
  21. package/dist/client.d.ts +164 -13
  22. package/dist/client.js +310 -163
  23. package/dist/client.js.map +1 -1
  24. package/dist/compactor.d.ts +1 -1
  25. package/dist/compactor.js +4 -4
  26. package/dist/connector-jira.d.ts +2 -2
  27. package/dist/connector-jira.js +3 -3
  28. package/dist/connector-salesforce.d.ts +2 -2
  29. package/dist/connector-salesforce.js +3 -3
  30. package/dist/{coordinator-DN8D8C7W.d.ts → coordinator-NXy6tA0h.d.ts} +23 -16
  31. package/dist/{db-types-B6_JKQWK.d.ts → db-types-CfLMUBfW.d.ts} +1 -1
  32. package/dist/gateway-server.d.ts +158 -64
  33. package/dist/gateway-server.js +482 -4003
  34. package/dist/gateway-server.js.map +1 -1
  35. package/dist/gateway.d.ts +61 -104
  36. package/dist/gateway.js +12 -6
  37. package/dist/index.d.ts +45 -10
  38. package/dist/index.js +14 -2
  39. package/dist/parquet.d.ts +1 -1
  40. package/dist/parquet.js +3 -3
  41. package/dist/proto.d.ts +1 -1
  42. package/dist/proto.js +3 -3
  43. package/dist/react.d.ts +47 -10
  44. package/dist/react.js +88 -40
  45. package/dist/react.js.map +1 -1
  46. package/dist/{registry-BN_9spxE.d.ts → registry-BcspAtZI.d.ts} +19 -4
  47. package/dist/{gateway-CvO7Xy3T.d.ts → request-handler-pUvL7ozF.d.ts} +139 -10
  48. package/dist/{resolver-BZURzdlL.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
  49. package/dist/{src-RR7I76OL.js → src-B6NLV3FP.js} +4 -4
  50. package/dist/{src-SLVE5567.js → src-ROW4XLO7.js} +15 -3
  51. package/dist/{src-V2CTPR7V.js → src-ZRHKG42A.js} +4 -4
  52. package/dist/{types-GGBfZBKQ.d.ts → types-BdGBv2ba.d.ts} +23 -2
  53. package/dist/{types-D-E0VrfS.d.ts → types-BrcD1oJg.d.ts} +26 -19
  54. package/package.json +1 -1
  55. package/dist/chunk-7D4SUZUM.js +0 -38
  56. package/dist/chunk-7UBS6MFH.js.map +0 -1
  57. package/dist/chunk-PWGQ3PXE.js.map +0 -1
  58. package/dist/chunk-Z7FGLEQU.js.map +0 -1
  59. /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
  60. /package/dist/{chunk-L4ZL5JA7.js.map → chunk-KVSWLIJR.js.map} +0 -0
  61. /package/dist/{chunk-46CKACNC.js.map → chunk-SSICS5KI.js.map} +0 -0
  62. /package/dist/{chunk-B3QEUG6E.js.map → chunk-TMLG32QV.js.map} +0 -0
  63. /package/dist/{src-RR7I76OL.js.map → src-B6NLV3FP.js.map} +0 -0
  64. /package/dist/{src-SLVE5567.js.map → src-ROW4XLO7.js.map} +0 -0
  65. /package/dist/{src-V2CTPR7V.js.map → src-ZRHKG42A.js.map} +0 -0
@@ -3,7 +3,7 @@ import {
3
3
  Err,
4
4
  Ok,
5
5
  toError
6
- } from "./chunk-7UBS6MFH.js";
6
+ } from "./chunk-LDFFCG2K.js";
7
7
 
8
8
  // ../adapter/src/db-types.ts
9
9
  var BIGQUERY_TYPE_MAP = {
@@ -20,32 +20,29 @@ function isDatabaseAdapter(adapter) {
20
20
  return adapter !== null && typeof adapter === "object" && "insertDeltas" in adapter && "queryDeltasSince" in adapter && typeof adapter.insertDeltas === "function";
21
21
  }
22
22
 
23
- // ../adapter/src/materialise.ts
24
- function isMaterialisable(adapter) {
25
- return adapter !== null && typeof adapter === "object" && "materialise" in adapter && typeof adapter.materialise === "function";
26
- }
27
- function groupDeltasByTable(deltas) {
28
- const result = /* @__PURE__ */ new Map();
29
- for (const delta of deltas) {
30
- let rowIds = result.get(delta.table);
31
- if (!rowIds) {
32
- rowIds = /* @__PURE__ */ new Set();
33
- result.set(delta.table, rowIds);
23
+ // ../adapter/src/shared.ts
24
+ function groupAndMerge(rows) {
25
+ const byRowId = /* @__PURE__ */ new Map();
26
+ for (const row of rows) {
27
+ let arr = byRowId.get(row.row_id);
28
+ if (!arr) {
29
+ arr = [];
30
+ byRowId.set(row.row_id, arr);
34
31
  }
35
- rowIds.add(delta.rowId);
32
+ arr.push(row);
36
33
  }
37
- return result;
38
- }
39
- function buildSchemaIndex(schemas) {
40
- const index = /* @__PURE__ */ new Map();
41
- for (const schema of schemas) {
42
- const key = schema.sourceTable ?? schema.table;
43
- index.set(key, schema);
34
+ const upserts = [];
35
+ const deleteIds = [];
36
+ for (const [rowId, group] of byRowId) {
37
+ const state = mergeLatestState(group);
38
+ if (state !== null) {
39
+ upserts.push({ rowId, state });
40
+ } else {
41
+ deleteIds.push(rowId);
42
+ }
44
43
  }
45
- return index;
44
+ return { upserts, deleteIds };
46
45
  }
47
-
48
- // ../adapter/src/shared.ts
49
46
  function toCause(error) {
50
47
  return error instanceof Error ? error : void 0;
51
48
  }
@@ -80,6 +77,72 @@ function mergeLatestState(rows) {
80
77
  return state;
81
78
  }
82
79
 
80
+ // ../adapter/src/materialise.ts
81
+ function isMaterialisable(adapter) {
82
+ return adapter !== null && typeof adapter === "object" && "materialise" in adapter && typeof adapter.materialise === "function";
83
+ }
84
+ function resolvePrimaryKey(schema) {
85
+ return schema.primaryKey ?? ["row_id"];
86
+ }
87
+ function resolveConflictColumns(schema) {
88
+ return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
89
+ }
90
+ function isSoftDelete(schema) {
91
+ return schema.softDelete !== false;
92
+ }
93
+ function groupDeltasByTable(deltas) {
94
+ const result = /* @__PURE__ */ new Map();
95
+ for (const delta of deltas) {
96
+ let rowIds = result.get(delta.table);
97
+ if (!rowIds) {
98
+ rowIds = /* @__PURE__ */ new Set();
99
+ result.set(delta.table, rowIds);
100
+ }
101
+ rowIds.add(delta.rowId);
102
+ }
103
+ return result;
104
+ }
105
+ function buildSchemaIndex(schemas) {
106
+ const index = /* @__PURE__ */ new Map();
107
+ for (const schema of schemas) {
108
+ const key = schema.sourceTable ?? schema.table;
109
+ index.set(key, schema);
110
+ }
111
+ return index;
112
+ }
113
+ async function executeMaterialise(executor, dialect, deltas, schemas) {
114
+ if (deltas.length === 0) {
115
+ return Ok(void 0);
116
+ }
117
+ return wrapAsync(async () => {
118
+ const grouped = groupDeltasByTable(deltas);
119
+ const schemaIndex = buildSchemaIndex(schemas);
120
+ for (const [tableName, rowIds] of grouped) {
121
+ const schema = schemaIndex.get(tableName);
122
+ if (!schema) continue;
123
+ const dest = schema.table;
124
+ const pk = resolvePrimaryKey(schema);
125
+ const conflictCols = resolveConflictColumns(schema);
126
+ const soft = isSoftDelete(schema);
127
+ const createStmt = dialect.createDestinationTable(dest, schema, pk, soft);
128
+ await executor.query(createStmt.sql, createStmt.params);
129
+ const sourceTable = schema.sourceTable ?? schema.table;
130
+ const rowIdArray = [...rowIds];
131
+ const historyStmt = dialect.queryDeltaHistory(sourceTable, rowIdArray);
132
+ const rows = await executor.queryRows(historyStmt.sql, historyStmt.params);
133
+ const { upserts, deleteIds } = groupAndMerge(rows);
134
+ if (upserts.length > 0) {
135
+ const upsertStmt = dialect.buildUpsert(dest, schema, conflictCols, soft, upserts);
136
+ await executor.query(upsertStmt.sql, upsertStmt.params);
137
+ }
138
+ if (deleteIds.length > 0) {
139
+ const deleteStmt = dialect.buildDelete(dest, deleteIds, soft);
140
+ await executor.query(deleteStmt.sql, deleteStmt.params);
141
+ }
142
+ }
143
+ }, "Failed to materialise deltas");
144
+ }
145
+
83
146
  // ../adapter/src/bigquery.ts
84
147
  import { BigQuery } from "@google-cloud/bigquery";
85
148
  function rowToRowDelta(row) {
@@ -96,6 +159,121 @@ function rowToRowDelta(row) {
96
159
  op: row.op
97
160
  };
98
161
  }
162
+ var BigQuerySqlDialect = class {
163
+ constructor(dataset) {
164
+ this.dataset = dataset;
165
+ }
166
+ createDestinationTable(dest, schema, pk, softDelete) {
167
+ const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
168
+ const deletedAtCol = softDelete ? `,
169
+ deleted_at TIMESTAMP` : "";
170
+ return {
171
+ sql: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${dest}\` (
172
+ row_id STRING NOT NULL,
173
+ ${colDefs},
174
+ props JSON DEFAULT '{}'${deletedAtCol},
175
+ synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
176
+ )
177
+ CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
178
+ params: []
179
+ };
180
+ }
181
+ queryDeltaHistory(sourceTable, rowIds) {
182
+ return {
183
+ sql: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
184
+ WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
185
+ ORDER BY hlc ASC`,
186
+ params: [
187
+ ["sourceTable", sourceTable],
188
+ ["rowIds", rowIds]
189
+ ]
190
+ };
191
+ }
192
+ buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
193
+ const namedParams = [];
194
+ const selects = [];
195
+ for (let i = 0; i < upserts.length; i++) {
196
+ const u = upserts[i];
197
+ namedParams.push([`rid_${i}`, u.rowId]);
198
+ for (const col of schema.columns) {
199
+ namedParams.push([`c${schema.columns.indexOf(col)}_${i}`, u.state[col.name] ?? null]);
200
+ }
201
+ const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
202
+ const deletedAtSelect = softDelete ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
203
+ selects.push(
204
+ `SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
205
+ );
206
+ }
207
+ const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
208
+ const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
209
+ const softUpdateExtra = softDelete ? ", deleted_at = s.deleted_at" : "";
210
+ const insertColsList = [
211
+ "row_id",
212
+ ...schema.columns.map((c) => c.name),
213
+ "props",
214
+ ...softDelete ? ["deleted_at"] : [],
215
+ "synced_at"
216
+ ].join(", ");
217
+ const insertValsList = [
218
+ "s.row_id",
219
+ ...schema.columns.map((c) => `s.${c.name}`),
220
+ "'{}'",
221
+ ...softDelete ? ["s.deleted_at"] : [],
222
+ "s.synced_at"
223
+ ].join(", ");
224
+ return {
225
+ sql: `MERGE \`${this.dataset}.${dest}\` AS t
226
+ USING (${selects.join(" UNION ALL ")}) AS s
227
+ ON ${mergeOn}
228
+ WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
229
+ WHEN NOT MATCHED THEN INSERT (${insertColsList})
230
+ VALUES (${insertValsList})`,
231
+ params: namedParams
232
+ };
233
+ }
234
+ buildDelete(dest, deleteIds, softDelete) {
235
+ if (softDelete) {
236
+ return {
237
+ sql: `UPDATE \`${this.dataset}.${dest}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
238
+ params: [["rowIds", deleteIds]]
239
+ };
240
+ }
241
+ return {
242
+ sql: `DELETE FROM \`${this.dataset}.${dest}\` WHERE row_id IN UNNEST(@rowIds)`,
243
+ params: [["rowIds", deleteIds]]
244
+ };
245
+ }
246
+ };
247
+ function createBigQueryExecutor(client, location) {
248
+ function toNamedParams(params) {
249
+ if (params.length === 0) return void 0;
250
+ const result = {};
251
+ for (const entry of params) {
252
+ const [key, value] = entry;
253
+ result[key] = value;
254
+ }
255
+ return result;
256
+ }
257
+ return {
258
+ async query(sql, params) {
259
+ const namedParams = toNamedParams(params);
260
+ await client.query({
261
+ query: sql,
262
+ params: namedParams,
263
+ location
264
+ });
265
+ },
266
+ async queryRows(sql, params) {
267
+ const namedParams = toNamedParams(params);
268
+ const [rows] = await client.query({
269
+ query: sql,
270
+ params: namedParams,
271
+ location
272
+ });
273
+ return rows;
274
+ }
275
+ };
276
+ }
99
277
  var BigQueryAdapter = class {
100
278
  /** @internal */
101
279
  client;
@@ -103,6 +281,8 @@ var BigQueryAdapter = class {
103
281
  dataset;
104
282
  /** @internal */
105
283
  location;
284
+ dialect;
285
+ executor;
106
286
  constructor(config) {
107
287
  this.client = new BigQuery({
108
288
  projectId: config.projectId,
@@ -110,6 +290,8 @@ var BigQueryAdapter = class {
110
290
  });
111
291
  this.dataset = config.dataset;
112
292
  this.location = config.location ?? "US";
293
+ this.dialect = new BigQuerySqlDialect(this.dataset);
294
+ this.executor = createBigQueryExecutor(this.client, this.location);
113
295
  }
114
296
  /**
115
297
  * Insert deltas into the database in a single batch.
@@ -222,106 +404,11 @@ CLUSTER BY \`table\`, hlc`,
222
404
  /**
223
405
  * Materialise deltas into destination tables.
224
406
  *
225
- * For each affected table, queries the full delta history for touched rows,
226
- * merges to latest state via column-level LWW, then upserts live rows and
227
- * deletes tombstoned rows. The consumer-owned `props` column is never
228
- * touched on UPDATE.
407
+ * Delegates to the shared `executeMaterialise` algorithm with the
408
+ * BigQuery SQL dialect.
229
409
  */
230
410
  async materialise(deltas, schemas) {
231
- if (deltas.length === 0) {
232
- return Ok(void 0);
233
- }
234
- return wrapAsync(async () => {
235
- const tableRowIds = groupDeltasByTable(deltas);
236
- const schemaIndex = buildSchemaIndex(schemas);
237
- for (const [sourceTable, rowIds] of tableRowIds) {
238
- const schema = schemaIndex.get(sourceTable);
239
- if (!schema) continue;
240
- const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
241
- await this.client.query({
242
- query: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${schema.table}\` (
243
- row_id STRING NOT NULL,
244
- ${colDefs},
245
- props JSON DEFAULT '{}',
246
- synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
247
- )`,
248
- location: this.location
249
- });
250
- const rowIdArray = [...rowIds];
251
- const [deltaRows] = await this.client.query({
252
- query: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
253
- WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
254
- ORDER BY hlc ASC`,
255
- params: { sourceTable, rowIds: rowIdArray },
256
- location: this.location
257
- });
258
- const rowGroups = /* @__PURE__ */ new Map();
259
- for (const row of deltaRows) {
260
- let group = rowGroups.get(row.row_id);
261
- if (!group) {
262
- group = [];
263
- rowGroups.set(row.row_id, group);
264
- }
265
- group.push({ columns: row.columns, op: row.op });
266
- }
267
- const upserts = [];
268
- const deleteRowIds = [];
269
- for (const [rowId, group] of rowGroups) {
270
- const state = mergeLatestState(group);
271
- if (state === null) {
272
- deleteRowIds.push(rowId);
273
- } else {
274
- upserts.push({ rowId, state });
275
- }
276
- }
277
- if (upserts.length > 0) {
278
- const params = {};
279
- const selects = [];
280
- for (let i = 0; i < upserts.length; i++) {
281
- const u = upserts[i];
282
- params[`rid_${i}`] = u.rowId;
283
- for (const col of schema.columns) {
284
- params[`c${schema.columns.indexOf(col)}_${i}`] = u.state[col.name] ?? null;
285
- }
286
- const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
287
- selects.push(
288
- `SELECT @rid_${i} AS row_id, ${colSelects}, CURRENT_TIMESTAMP() AS synced_at`
289
- );
290
- }
291
- const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
292
- const insertCols = [
293
- "row_id",
294
- ...schema.columns.map((c) => c.name),
295
- "props",
296
- "synced_at"
297
- ].join(", ");
298
- const insertVals = [
299
- "s.row_id",
300
- ...schema.columns.map((c) => `s.${c.name}`),
301
- "'{}'",
302
- "s.synced_at"
303
- ].join(", ");
304
- const mergeSql = `MERGE \`${this.dataset}.${schema.table}\` AS t
305
- USING (${selects.join(" UNION ALL ")}) AS s
306
- ON t.row_id = s.row_id
307
- WHEN MATCHED THEN UPDATE SET ${updateSet}, synced_at = s.synced_at
308
- WHEN NOT MATCHED THEN INSERT (${insertCols})
309
- VALUES (${insertVals})`;
310
- await this.client.query({
311
- query: mergeSql,
312
- params,
313
- location: this.location
314
- });
315
- }
316
- if (deleteRowIds.length > 0) {
317
- await this.client.query({
318
- query: `DELETE FROM \`${this.dataset}.${schema.table}\` WHERE row_id IN UNNEST(@rowIds)`,
319
- params: { rowIds: deleteRowIds },
320
- location: this.location
321
- });
322
- }
323
- }
324
- }, "Failed to materialise deltas");
411
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
325
412
  }
326
413
  /**
327
414
  * No-op — BigQuery client is HTTP-based with no persistent connections.
@@ -432,12 +519,75 @@ var MYSQL_TYPE_MAP = {
432
519
  function lakeSyncTypeToMySQL(type) {
433
520
  return MYSQL_TYPE_MAP[type];
434
521
  }
522
+ var MySqlDialect = class {
523
+ createDestinationTable(dest, schema, pk, softDelete) {
524
+ const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
525
+ const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
526
+ const deletedAtCol = softDelete ? `, deleted_at TIMESTAMP NULL` : "";
527
+ const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
528
+ return {
529
+ sql: `CREATE TABLE IF NOT EXISTS \`${dest}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`,
530
+ params: []
531
+ };
532
+ }
533
+ queryDeltaHistory(sourceTable, rowIds) {
534
+ const placeholders = rowIds.map(() => "?").join(", ");
535
+ return {
536
+ sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
537
+ params: [sourceTable, ...rowIds]
538
+ };
539
+ }
540
+ buildUpsert(dest, schema, _conflictCols, softDelete, upserts) {
541
+ const cols = schema.columns.map((c) => c.name);
542
+ const valuePlaceholders = softDelete ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
543
+ const values = [];
544
+ for (const { rowId, state } of upserts) {
545
+ values.push(rowId);
546
+ for (const col of cols) {
547
+ values.push(state[col] ?? null);
548
+ }
549
+ }
550
+ const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
551
+ const softUpdateExtra = softDelete ? ", deleted_at = NULL" : "";
552
+ const colList = softDelete ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
553
+ return {
554
+ sql: `INSERT INTO \`${dest}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
555
+ params: values
556
+ };
557
+ }
558
+ buildDelete(dest, deleteIds, softDelete) {
559
+ const placeholders = deleteIds.map(() => "?").join(", ");
560
+ if (softDelete) {
561
+ return {
562
+ sql: `UPDATE \`${dest}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${placeholders})`,
563
+ params: deleteIds
564
+ };
565
+ }
566
+ return {
567
+ sql: `DELETE FROM \`${dest}\` WHERE row_id IN (${placeholders})`,
568
+ params: deleteIds
569
+ };
570
+ }
571
+ };
435
572
  var MySQLAdapter = class {
436
573
  /** @internal */
437
574
  pool;
575
+ dialect = new MySqlDialect();
438
576
  constructor(config) {
439
577
  this.pool = mysql.createPool(config.connectionString);
440
578
  }
579
+ get executor() {
580
+ const pool = this.pool;
581
+ return {
582
+ async query(sql, params) {
583
+ await pool.execute(sql, params);
584
+ },
585
+ async queryRows(sql, params) {
586
+ const [rows] = await pool.execute(sql, params);
587
+ return rows;
588
+ }
589
+ };
590
+ }
441
591
  /**
442
592
  * Insert deltas into the database in a single batch.
443
593
  * Uses INSERT IGNORE for idempotent writes — duplicate deltaIds are silently skipped.
@@ -519,74 +669,11 @@ var MySQLAdapter = class {
519
669
  /**
520
670
  * Materialise deltas into destination tables.
521
671
  *
522
- * For each table with a matching schema, merges delta history into the
523
- * latest row state and upserts into the destination table. Tombstoned
524
- * rows are deleted. The `props` column is never touched.
672
+ * Delegates to the shared `executeMaterialise` algorithm with the
673
+ * MySQL SQL dialect.
525
674
  */
526
675
  async materialise(deltas, schemas) {
527
- if (deltas.length === 0) {
528
- return Ok(void 0);
529
- }
530
- return wrapAsync(async () => {
531
- const grouped = groupDeltasByTable(deltas);
532
- const schemaIndex = buildSchemaIndex(schemas);
533
- for (const [tableName, rowIds] of grouped) {
534
- const schema = schemaIndex.get(tableName);
535
- if (!schema) continue;
536
- const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
537
- await this.pool.execute(
538
- `CREATE TABLE IF NOT EXISTS \`${schema.table}\` (row_id VARCHAR(255) PRIMARY KEY, ${typedCols}, props JSON NOT NULL DEFAULT ('{}'), synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)`
539
- );
540
- const rowIdArray = [...rowIds];
541
- const placeholders = rowIdArray.map(() => "?").join(", ");
542
- const [rows] = await this.pool.execute(
543
- `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
544
- [tableName, ...rowIdArray]
545
- );
546
- const byRow = /* @__PURE__ */ new Map();
547
- for (const row of rows) {
548
- let list = byRow.get(row.row_id);
549
- if (!list) {
550
- list = [];
551
- byRow.set(row.row_id, list);
552
- }
553
- list.push(row);
554
- }
555
- const upserts = [];
556
- const deleteIds = [];
557
- for (const [rowId, rowDeltas] of byRow) {
558
- const state = mergeLatestState(rowDeltas);
559
- if (state === null) {
560
- deleteIds.push(rowId);
561
- } else {
562
- upserts.push({ rowId, state });
563
- }
564
- }
565
- if (upserts.length > 0) {
566
- const cols = schema.columns.map((c) => c.name);
567
- const valuePlaceholders = upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
568
- const values = [];
569
- for (const { rowId, state } of upserts) {
570
- values.push(rowId);
571
- for (const col of cols) {
572
- values.push(state[col] ?? null);
573
- }
574
- }
575
- const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
576
- await this.pool.execute(
577
- `INSERT INTO \`${schema.table}\` (row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}, synced_at = VALUES(synced_at)`,
578
- values
579
- );
580
- }
581
- if (deleteIds.length > 0) {
582
- const delPlaceholders = deleteIds.map(() => "?").join(", ");
583
- await this.pool.execute(
584
- `DELETE FROM \`${schema.table}\` WHERE row_id IN (${delPlaceholders})`,
585
- deleteIds
586
- );
587
- }
588
- }
589
- }, "Failed to materialise deltas");
676
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
590
677
  }
591
678
  /** Close the database connection pool and release resources. */
592
679
  async close() {
@@ -614,15 +701,94 @@ var POSTGRES_TYPE_MAP = {
614
701
  json: "JSONB",
615
702
  null: "TEXT"
616
703
  };
704
+ var PostgresSqlDialect = class {
705
+ createDestinationTable(dest, schema, pk, softDelete) {
706
+ const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
707
+ const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
708
+ const deletedAtCol = softDelete ? `,
709
+ deleted_at TIMESTAMPTZ` : "";
710
+ const uniqueConstraint = schema.externalIdColumn ? `,
711
+ UNIQUE ("${schema.externalIdColumn}")` : "";
712
+ return {
713
+ sql: `CREATE TABLE IF NOT EXISTS "${dest}" (
714
+ row_id TEXT NOT NULL,
715
+ ${columnDefs},
716
+ props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
717
+ synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
718
+ ${pkConstraint}${uniqueConstraint}
719
+ )`,
720
+ params: []
721
+ };
722
+ }
723
+ queryDeltaHistory(sourceTable, rowIds) {
724
+ return {
725
+ sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
726
+ params: [sourceTable, rowIds]
727
+ };
728
+ }
729
+ buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
730
+ const colNames = schema.columns.map((c) => c.name);
731
+ const baseCols = ["row_id", ...colNames];
732
+ const allCols = softDelete ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
733
+ const colList = allCols.map((c) => `"${c}"`).join(", ");
734
+ const values = [];
735
+ const valueRows = [];
736
+ const paramsPerRow = allCols.length;
737
+ for (let i = 0; i < upserts.length; i++) {
738
+ const u = upserts[i];
739
+ const offset = i * paramsPerRow;
740
+ const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
741
+ valueRows.push(`(${placeholders.join(", ")})`);
742
+ values.push(u.rowId);
743
+ for (const col of colNames) {
744
+ values.push(u.state[col] ?? null);
745
+ }
746
+ if (softDelete) values.push(null);
747
+ values.push(/* @__PURE__ */ new Date());
748
+ }
749
+ const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
750
+ const updateCols = softDelete ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
751
+ const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
752
+ return {
753
+ sql: `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
754
+ params: values
755
+ };
756
+ }
757
+ buildDelete(dest, deleteIds, softDelete) {
758
+ if (softDelete) {
759
+ return {
760
+ sql: `UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
761
+ params: [deleteIds]
762
+ };
763
+ }
764
+ return {
765
+ sql: `DELETE FROM "${dest}" WHERE row_id = ANY($1)`,
766
+ params: [deleteIds]
767
+ };
768
+ }
769
+ };
617
770
  var PostgresAdapter = class {
618
771
  /** @internal */
619
772
  pool;
773
+ dialect = new PostgresSqlDialect();
620
774
  constructor(config) {
621
775
  const poolConfig = {
622
776
  connectionString: config.connectionString
623
777
  };
624
778
  this.pool = new Pool(poolConfig);
625
779
  }
780
+ get executor() {
781
+ const pool = this.pool;
782
+ return {
783
+ async query(sql, params) {
784
+ await pool.query(sql, params);
785
+ },
786
+ async queryRows(sql, params) {
787
+ const result = await pool.query(sql, params);
788
+ return result.rows;
789
+ }
790
+ };
791
+ }
626
792
  /**
627
793
  * Insert deltas into the database in a single batch.
628
794
  * Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
@@ -719,85 +885,11 @@ CREATE INDEX IF NOT EXISTS idx_lakesync_deltas_table_row ON lakesync_deltas ("ta
719
885
  /**
720
886
  * Materialise deltas into destination tables.
721
887
  *
722
- * For each table with a matching schema, merges delta history into the
723
- * latest row state and upserts into the destination table. Tombstoned
724
- * rows are deleted. The `props` column is never touched.
888
+ * Delegates to the shared `executeMaterialise` algorithm with the
889
+ * Postgres SQL dialect.
725
890
  */
726
891
  async materialise(deltas, schemas) {
727
- if (deltas.length === 0) {
728
- return Ok(void 0);
729
- }
730
- return wrapAsync(async () => {
731
- const grouped = groupDeltasByTable(deltas);
732
- const schemaIndex = buildSchemaIndex(schemas);
733
- for (const [tableName, rowIds] of grouped) {
734
- const schema = schemaIndex.get(tableName);
735
- if (!schema) continue;
736
- const dest = schema.table;
737
- const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
738
- await this.pool.query(
739
- `CREATE TABLE IF NOT EXISTS "${dest}" (
740
- row_id TEXT PRIMARY KEY,
741
- ${columnDefs},
742
- props JSONB NOT NULL DEFAULT '{}',
743
- synced_at TIMESTAMPTZ NOT NULL DEFAULT now()
744
- )`
745
- );
746
- const sourceTable = schema.sourceTable ?? schema.table;
747
- const rowIdArray = [...rowIds];
748
- const deltaResult = await this.pool.query(
749
- `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
750
- [sourceTable, rowIdArray]
751
- );
752
- const byRowId = /* @__PURE__ */ new Map();
753
- for (const row of deltaResult.rows) {
754
- const rid = row.row_id;
755
- let arr = byRowId.get(rid);
756
- if (!arr) {
757
- arr = [];
758
- byRowId.set(rid, arr);
759
- }
760
- arr.push(row);
761
- }
762
- const upserts = [];
763
- const deleteIds = [];
764
- for (const [rowId, rows] of byRowId) {
765
- const state = mergeLatestState(rows);
766
- if (state !== null) {
767
- upserts.push({ rowId, state });
768
- } else {
769
- deleteIds.push(rowId);
770
- }
771
- }
772
- if (upserts.length > 0) {
773
- const colNames = schema.columns.map((c) => c.name);
774
- const allCols = ["row_id", ...colNames, "synced_at"];
775
- const colList = allCols.map((c) => `"${c}"`).join(", ");
776
- const values = [];
777
- const valueRows = [];
778
- const paramsPerRow = allCols.length;
779
- for (let i = 0; i < upserts.length; i++) {
780
- const u = upserts[i];
781
- const offset = i * paramsPerRow;
782
- const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
783
- valueRows.push(`(${placeholders.join(", ")})`);
784
- values.push(u.rowId);
785
- for (const col of colNames) {
786
- values.push(u.state[col] ?? null);
787
- }
788
- values.push(/* @__PURE__ */ new Date());
789
- }
790
- const updateSet = [...colNames, "synced_at"].map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
791
- await this.pool.query(
792
- `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (row_id) DO UPDATE SET ${updateSet}`,
793
- values
794
- );
795
- }
796
- if (deleteIds.length > 0) {
797
- await this.pool.query(`DELETE FROM "${dest}" WHERE row_id = ANY($1)`, [deleteIds]);
798
- }
799
- }
800
- }, "Failed to materialise deltas");
892
+ return executeMaterialise(this.executor, this.dialect, deltas, schemas);
801
893
  }
802
894
  /** Close the database connection pool and release resources. */
803
895
  async close() {
@@ -821,30 +913,19 @@ function rowToRowDelta2(row) {
821
913
  function createDatabaseAdapter(config) {
822
914
  try {
823
915
  switch (config.type) {
824
- case "postgres": {
825
- if (!config.postgres) {
826
- return Err(new AdapterError("Postgres connector config missing postgres field"));
827
- }
916
+ case "postgres":
828
917
  return Ok(
829
918
  new PostgresAdapter({
830
919
  connectionString: config.postgres.connectionString
831
920
  })
832
921
  );
833
- }
834
- case "mysql": {
835
- if (!config.mysql) {
836
- return Err(new AdapterError("MySQL connector config missing mysql field"));
837
- }
922
+ case "mysql":
838
923
  return Ok(
839
924
  new MySQLAdapter({
840
925
  connectionString: config.mysql.connectionString
841
926
  })
842
927
  );
843
- }
844
- case "bigquery": {
845
- if (!config.bigquery) {
846
- return Err(new AdapterError("BigQuery connector config missing bigquery field"));
847
- }
928
+ case "bigquery":
848
929
  return Ok(
849
930
  new BigQueryAdapter({
850
931
  projectId: config.bigquery.projectId,
@@ -853,9 +934,11 @@ function createDatabaseAdapter(config) {
853
934
  location: config.bigquery.location
854
935
  })
855
936
  );
856
- }
857
- default:
858
- return Err(new AdapterError(`Unsupported connector type: ${config.type}`));
937
+ case "jira":
938
+ case "salesforce":
939
+ return Err(
940
+ new AdapterError(`Connector type "${config.type}" does not use a DatabaseAdapter`)
941
+ );
859
942
  }
860
943
  } catch (err) {
861
944
  return Err(new AdapterError(`Failed to create adapter: ${toError(err).message}`));
@@ -1151,7 +1234,6 @@ var MinIOAdapter = class {
1151
1234
  async function createQueryFn(config) {
1152
1235
  switch (config.type) {
1153
1236
  case "postgres": {
1154
- if (!config.postgres) return null;
1155
1237
  const { Pool: Pool2 } = await import("pg");
1156
1238
  const pool = new Pool2({ connectionString: config.postgres.connectionString });
1157
1239
  return async (sql, params) => {
@@ -1160,7 +1242,6 @@ async function createQueryFn(config) {
1160
1242
  };
1161
1243
  }
1162
1244
  case "mysql": {
1163
- if (!config.mysql) return null;
1164
1245
  const mysql2 = await import("mysql2/promise");
1165
1246
  const pool = mysql2.createPool(config.mysql.connectionString);
1166
1247
  return async (sql, params) => {
@@ -1176,15 +1257,23 @@ async function createQueryFn(config) {
1176
1257
  export {
1177
1258
  lakeSyncTypeToBigQuery,
1178
1259
  isDatabaseAdapter,
1179
- isMaterialisable,
1180
- groupDeltasByTable,
1181
- buildSchemaIndex,
1260
+ groupAndMerge,
1182
1261
  toCause,
1183
1262
  wrapAsync,
1184
1263
  mergeLatestState,
1264
+ isMaterialisable,
1265
+ resolvePrimaryKey,
1266
+ resolveConflictColumns,
1267
+ isSoftDelete,
1268
+ groupDeltasByTable,
1269
+ buildSchemaIndex,
1270
+ executeMaterialise,
1271
+ BigQuerySqlDialect,
1185
1272
  BigQueryAdapter,
1186
1273
  CompositeAdapter,
1274
+ MySqlDialect,
1187
1275
  MySQLAdapter,
1276
+ PostgresSqlDialect,
1188
1277
  PostgresAdapter,
1189
1278
  createDatabaseAdapter,
1190
1279
  FanOutAdapter,
@@ -1194,4 +1283,4 @@ export {
1194
1283
  MinIOAdapter,
1195
1284
  createQueryFn
1196
1285
  };
1197
- //# sourceMappingURL=chunk-Z7FGLEQU.js.map
1286
+ //# sourceMappingURL=chunk-LPWXOYNS.js.map