lakesync 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-types-DwsQGQS4.d.ts +94 -0
- package/dist/adapter.d.ts +202 -63
- package/dist/adapter.js +20 -5
- package/dist/analyst.js +2 -2
- package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Y7ORYgUv.d.ts} +78 -19
- package/dist/catalogue.d.ts +1 -1
- package/dist/catalogue.js +3 -3
- package/dist/{chunk-P3FT7QCW.js → chunk-4SG66H5K.js} +395 -252
- package/dist/chunk-4SG66H5K.js.map +1 -0
- package/dist/{chunk-GUJWMK5P.js → chunk-C4KD6YKP.js} +419 -380
- package/dist/chunk-C4KD6YKP.js.map +1 -0
- package/dist/chunk-DGUM43GV.js +11 -0
- package/dist/{chunk-IRJ4QRWV.js → chunk-FIIHPQMQ.js} +396 -209
- package/dist/chunk-FIIHPQMQ.js.map +1 -0
- package/dist/{chunk-UAUQGP3B.js → chunk-U2NV4DUX.js} +2 -2
- package/dist/{chunk-NCZYFZ3B.js → chunk-XVP5DJJ7.js} +44 -18
- package/dist/{chunk-NCZYFZ3B.js.map → chunk-XVP5DJJ7.js.map} +1 -1
- package/dist/{chunk-FHVTUKXL.js → chunk-YHYBLU6W.js} +2 -2
- package/dist/{chunk-QMS7TGFL.js → chunk-ZNY4DSFU.js} +29 -15
- package/dist/{chunk-QMS7TGFL.js.map → chunk-ZNY4DSFU.js.map} +1 -1
- package/dist/{chunk-SF7Y6ZUA.js → chunk-ZU7RC7CT.js} +2 -2
- package/dist/client.d.ts +186 -17
- package/dist/client.js +456 -188
- package/dist/client.js.map +1 -1
- package/dist/compactor.d.ts +2 -2
- package/dist/compactor.js +4 -4
- package/dist/connector-jira.d.ts +13 -3
- package/dist/connector-jira.js +7 -3
- package/dist/connector-salesforce.d.ts +13 -3
- package/dist/connector-salesforce.js +7 -3
- package/dist/{coordinator-D32a5rNk.d.ts → coordinator-eGmZMnJ_.d.ts} +120 -30
- package/dist/create-poller-Cc2MGfhh.d.ts +55 -0
- package/dist/factory-DFfR-030.d.ts +33 -0
- package/dist/gateway-server.d.ts +516 -119
- package/dist/gateway-server.js +1201 -4035
- package/dist/gateway-server.js.map +1 -1
- package/dist/gateway.d.ts +69 -106
- package/dist/gateway.js +13 -6
- package/dist/index.d.ts +65 -58
- package/dist/index.js +18 -4
- package/dist/parquet.d.ts +1 -1
- package/dist/parquet.js +3 -3
- package/dist/proto.d.ts +1 -1
- package/dist/proto.js +3 -3
- package/dist/react.d.ts +47 -10
- package/dist/react.js +88 -40
- package/dist/react.js.map +1 -1
- package/dist/{registry-CPTgO9jv.d.ts → registry-Dd8JuW8T.d.ts} +19 -4
- package/dist/{gateway-Bpvatd9n.d.ts → request-handler-B1I5xDOx.d.ts} +193 -20
- package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
- package/dist/{src-RHKJFQKR.js → src-WU7IBVC4.js} +19 -5
- package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
- package/dist/{types-D-E0VrfS.d.ts → types-D2C9jTbL.d.ts} +39 -22
- package/package.json +1 -1
- package/dist/auth-CAVutXzx.d.ts +0 -30
- package/dist/chunk-7D4SUZUM.js +0 -38
- package/dist/chunk-GUJWMK5P.js.map +0 -1
- package/dist/chunk-IRJ4QRWV.js.map +0 -1
- package/dist/chunk-P3FT7QCW.js.map +0 -1
- package/dist/db-types-BlN-4KbQ.d.ts +0 -29
- package/dist/src-CLCALYDT.js +0 -25
- package/dist/src-FPJQYQNA.js +0 -27
- package/dist/src-FPJQYQNA.js.map +0 -1
- package/dist/src-RHKJFQKR.js.map +0 -1
- package/dist/types-DSC_EiwR.d.ts +0 -45
- /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
- /package/dist/{chunk-UAUQGP3B.js.map → chunk-U2NV4DUX.js.map} +0 -0
- /package/dist/{chunk-FHVTUKXL.js.map → chunk-YHYBLU6W.js.map} +0 -0
- /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-ZU7RC7CT.js.map} +0 -0
- /package/dist/{src-CLCALYDT.js.map → src-WU7IBVC4.js.map} +0 -0
|
@@ -2,8 +2,9 @@ import {
|
|
|
2
2
|
AdapterError,
|
|
3
3
|
Err,
|
|
4
4
|
Ok,
|
|
5
|
+
isMaterialisable,
|
|
5
6
|
toError
|
|
6
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-4SG66H5K.js";
|
|
7
8
|
|
|
8
9
|
// ../adapter/src/db-types.ts
|
|
9
10
|
var BIGQUERY_TYPE_MAP = {
|
|
@@ -16,45 +17,30 @@ var BIGQUERY_TYPE_MAP = {
|
|
|
16
17
|
function lakeSyncTypeToBigQuery(type) {
|
|
17
18
|
return BIGQUERY_TYPE_MAP[type];
|
|
18
19
|
}
|
|
19
|
-
function isDatabaseAdapter(adapter) {
|
|
20
|
-
return adapter !== null && typeof adapter === "object" && "insertDeltas" in adapter && "queryDeltasSince" in adapter && typeof adapter.insertDeltas === "function";
|
|
21
|
-
}
|
|
22
20
|
|
|
23
|
-
// ../adapter/src/
|
|
24
|
-
function
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
|
|
32
|
-
}
|
|
33
|
-
function isSoftDelete(schema) {
|
|
34
|
-
return schema.softDelete !== false;
|
|
35
|
-
}
|
|
36
|
-
function groupDeltasByTable(deltas) {
|
|
37
|
-
const result = /* @__PURE__ */ new Map();
|
|
38
|
-
for (const delta of deltas) {
|
|
39
|
-
let rowIds = result.get(delta.table);
|
|
40
|
-
if (!rowIds) {
|
|
41
|
-
rowIds = /* @__PURE__ */ new Set();
|
|
42
|
-
result.set(delta.table, rowIds);
|
|
21
|
+
// ../adapter/src/shared.ts
|
|
22
|
+
function groupAndMerge(rows) {
|
|
23
|
+
const byRowId = /* @__PURE__ */ new Map();
|
|
24
|
+
for (const row of rows) {
|
|
25
|
+
let arr = byRowId.get(row.row_id);
|
|
26
|
+
if (!arr) {
|
|
27
|
+
arr = [];
|
|
28
|
+
byRowId.set(row.row_id, arr);
|
|
43
29
|
}
|
|
44
|
-
|
|
30
|
+
arr.push(row);
|
|
45
31
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
32
|
+
const upserts = [];
|
|
33
|
+
const deleteIds = [];
|
|
34
|
+
for (const [rowId, group] of byRowId) {
|
|
35
|
+
const state = mergeLatestState(group);
|
|
36
|
+
if (state !== null) {
|
|
37
|
+
upserts.push({ rowId, state });
|
|
38
|
+
} else {
|
|
39
|
+
deleteIds.push(rowId);
|
|
40
|
+
}
|
|
53
41
|
}
|
|
54
|
-
return
|
|
42
|
+
return { upserts, deleteIds };
|
|
55
43
|
}
|
|
56
|
-
|
|
57
|
-
// ../adapter/src/shared.ts
|
|
58
44
|
function toCause(error) {
|
|
59
45
|
return error instanceof Error ? error : void 0;
|
|
60
46
|
}
|
|
@@ -89,6 +75,69 @@ function mergeLatestState(rows) {
|
|
|
89
75
|
return state;
|
|
90
76
|
}
|
|
91
77
|
|
|
78
|
+
// ../adapter/src/materialise.ts
|
|
79
|
+
function resolvePrimaryKey(schema) {
|
|
80
|
+
return schema.primaryKey ?? ["row_id"];
|
|
81
|
+
}
|
|
82
|
+
function resolveConflictColumns(schema) {
|
|
83
|
+
return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
|
|
84
|
+
}
|
|
85
|
+
function isSoftDelete(schema) {
|
|
86
|
+
return schema.softDelete !== false;
|
|
87
|
+
}
|
|
88
|
+
function groupDeltasByTable(deltas) {
|
|
89
|
+
const result = /* @__PURE__ */ new Map();
|
|
90
|
+
for (const delta of deltas) {
|
|
91
|
+
let rowIds = result.get(delta.table);
|
|
92
|
+
if (!rowIds) {
|
|
93
|
+
rowIds = /* @__PURE__ */ new Set();
|
|
94
|
+
result.set(delta.table, rowIds);
|
|
95
|
+
}
|
|
96
|
+
rowIds.add(delta.rowId);
|
|
97
|
+
}
|
|
98
|
+
return result;
|
|
99
|
+
}
|
|
100
|
+
function buildSchemaIndex(schemas) {
|
|
101
|
+
const index = /* @__PURE__ */ new Map();
|
|
102
|
+
for (const schema of schemas) {
|
|
103
|
+
const key = schema.sourceTable ?? schema.table;
|
|
104
|
+
index.set(key, schema);
|
|
105
|
+
}
|
|
106
|
+
return index;
|
|
107
|
+
}
|
|
108
|
+
async function executeMaterialise(executor, dialect, deltas, schemas) {
|
|
109
|
+
if (deltas.length === 0) {
|
|
110
|
+
return Ok(void 0);
|
|
111
|
+
}
|
|
112
|
+
return wrapAsync(async () => {
|
|
113
|
+
const grouped = groupDeltasByTable(deltas);
|
|
114
|
+
const schemaIndex = buildSchemaIndex(schemas);
|
|
115
|
+
for (const [tableName, rowIds] of grouped) {
|
|
116
|
+
const schema = schemaIndex.get(tableName);
|
|
117
|
+
if (!schema) continue;
|
|
118
|
+
const dest = schema.table;
|
|
119
|
+
const pk = resolvePrimaryKey(schema);
|
|
120
|
+
const conflictCols = resolveConflictColumns(schema);
|
|
121
|
+
const soft = isSoftDelete(schema);
|
|
122
|
+
const createStmt = dialect.createDestinationTable(dest, schema, pk, soft);
|
|
123
|
+
await executor.query(createStmt.sql, createStmt.params);
|
|
124
|
+
const sourceTable = schema.sourceTable ?? schema.table;
|
|
125
|
+
const rowIdArray = [...rowIds];
|
|
126
|
+
const historyStmt = dialect.queryDeltaHistory(sourceTable, rowIdArray);
|
|
127
|
+
const rows = await executor.queryRows(historyStmt.sql, historyStmt.params);
|
|
128
|
+
const { upserts, deleteIds } = groupAndMerge(rows);
|
|
129
|
+
if (upserts.length > 0) {
|
|
130
|
+
const upsertStmt = dialect.buildUpsert(dest, schema, conflictCols, soft, upserts);
|
|
131
|
+
await executor.query(upsertStmt.sql, upsertStmt.params);
|
|
132
|
+
}
|
|
133
|
+
if (deleteIds.length > 0) {
|
|
134
|
+
const deleteStmt = dialect.buildDelete(dest, deleteIds, soft);
|
|
135
|
+
await executor.query(deleteStmt.sql, deleteStmt.params);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}, "Failed to materialise deltas");
|
|
139
|
+
}
|
|
140
|
+
|
|
92
141
|
// ../adapter/src/bigquery.ts
|
|
93
142
|
import { BigQuery } from "@google-cloud/bigquery";
|
|
94
143
|
function rowToRowDelta(row) {
|
|
@@ -105,6 +154,121 @@ function rowToRowDelta(row) {
|
|
|
105
154
|
op: row.op
|
|
106
155
|
};
|
|
107
156
|
}
|
|
157
|
+
var BigQuerySqlDialect = class {
|
|
158
|
+
constructor(dataset) {
|
|
159
|
+
this.dataset = dataset;
|
|
160
|
+
}
|
|
161
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
162
|
+
const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
|
|
163
|
+
const deletedAtCol = softDelete ? `,
|
|
164
|
+
deleted_at TIMESTAMP` : "";
|
|
165
|
+
return {
|
|
166
|
+
sql: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${dest}\` (
|
|
167
|
+
row_id STRING NOT NULL,
|
|
168
|
+
${colDefs},
|
|
169
|
+
props JSON DEFAULT '{}'${deletedAtCol},
|
|
170
|
+
synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
|
|
171
|
+
)
|
|
172
|
+
CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
|
|
173
|
+
params: []
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
177
|
+
return {
|
|
178
|
+
sql: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
|
|
179
|
+
WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
|
|
180
|
+
ORDER BY hlc ASC`,
|
|
181
|
+
params: [
|
|
182
|
+
["sourceTable", sourceTable],
|
|
183
|
+
["rowIds", rowIds]
|
|
184
|
+
]
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
|
|
188
|
+
const namedParams = [];
|
|
189
|
+
const selects = [];
|
|
190
|
+
for (let i = 0; i < upserts.length; i++) {
|
|
191
|
+
const u = upserts[i];
|
|
192
|
+
namedParams.push([`rid_${i}`, u.rowId]);
|
|
193
|
+
for (const col of schema.columns) {
|
|
194
|
+
namedParams.push([`c${schema.columns.indexOf(col)}_${i}`, u.state[col.name] ?? null]);
|
|
195
|
+
}
|
|
196
|
+
const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
|
|
197
|
+
const deletedAtSelect = softDelete ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
|
|
198
|
+
selects.push(
|
|
199
|
+
`SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
|
|
203
|
+
const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
|
|
204
|
+
const softUpdateExtra = softDelete ? ", deleted_at = s.deleted_at" : "";
|
|
205
|
+
const insertColsList = [
|
|
206
|
+
"row_id",
|
|
207
|
+
...schema.columns.map((c) => c.name),
|
|
208
|
+
"props",
|
|
209
|
+
...softDelete ? ["deleted_at"] : [],
|
|
210
|
+
"synced_at"
|
|
211
|
+
].join(", ");
|
|
212
|
+
const insertValsList = [
|
|
213
|
+
"s.row_id",
|
|
214
|
+
...schema.columns.map((c) => `s.${c.name}`),
|
|
215
|
+
"'{}'",
|
|
216
|
+
...softDelete ? ["s.deleted_at"] : [],
|
|
217
|
+
"s.synced_at"
|
|
218
|
+
].join(", ");
|
|
219
|
+
return {
|
|
220
|
+
sql: `MERGE \`${this.dataset}.${dest}\` AS t
|
|
221
|
+
USING (${selects.join(" UNION ALL ")}) AS s
|
|
222
|
+
ON ${mergeOn}
|
|
223
|
+
WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
|
|
224
|
+
WHEN NOT MATCHED THEN INSERT (${insertColsList})
|
|
225
|
+
VALUES (${insertValsList})`,
|
|
226
|
+
params: namedParams
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
230
|
+
if (softDelete) {
|
|
231
|
+
return {
|
|
232
|
+
sql: `UPDATE \`${this.dataset}.${dest}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
|
|
233
|
+
params: [["rowIds", deleteIds]]
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
sql: `DELETE FROM \`${this.dataset}.${dest}\` WHERE row_id IN UNNEST(@rowIds)`,
|
|
238
|
+
params: [["rowIds", deleteIds]]
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
function createBigQueryExecutor(client, location) {
|
|
243
|
+
function toNamedParams(params) {
|
|
244
|
+
if (params.length === 0) return void 0;
|
|
245
|
+
const result = {};
|
|
246
|
+
for (const entry of params) {
|
|
247
|
+
const [key, value] = entry;
|
|
248
|
+
result[key] = value;
|
|
249
|
+
}
|
|
250
|
+
return result;
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
async query(sql, params) {
|
|
254
|
+
const namedParams = toNamedParams(params);
|
|
255
|
+
await client.query({
|
|
256
|
+
query: sql,
|
|
257
|
+
params: namedParams,
|
|
258
|
+
location
|
|
259
|
+
});
|
|
260
|
+
},
|
|
261
|
+
async queryRows(sql, params) {
|
|
262
|
+
const namedParams = toNamedParams(params);
|
|
263
|
+
const [rows] = await client.query({
|
|
264
|
+
query: sql,
|
|
265
|
+
params: namedParams,
|
|
266
|
+
location
|
|
267
|
+
});
|
|
268
|
+
return rows;
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
}
|
|
108
272
|
var BigQueryAdapter = class {
|
|
109
273
|
/** @internal */
|
|
110
274
|
client;
|
|
@@ -112,6 +276,8 @@ var BigQueryAdapter = class {
|
|
|
112
276
|
dataset;
|
|
113
277
|
/** @internal */
|
|
114
278
|
location;
|
|
279
|
+
dialect;
|
|
280
|
+
executor;
|
|
115
281
|
constructor(config) {
|
|
116
282
|
this.client = new BigQuery({
|
|
117
283
|
projectId: config.projectId,
|
|
@@ -119,6 +285,8 @@ var BigQueryAdapter = class {
|
|
|
119
285
|
});
|
|
120
286
|
this.dataset = config.dataset;
|
|
121
287
|
this.location = config.location ?? "US";
|
|
288
|
+
this.dialect = new BigQuerySqlDialect(this.dataset);
|
|
289
|
+
this.executor = createBigQueryExecutor(this.client, this.location);
|
|
122
290
|
}
|
|
123
291
|
/**
|
|
124
292
|
* Insert deltas into the database in a single batch.
|
|
@@ -231,125 +399,11 @@ CLUSTER BY \`table\`, hlc`,
|
|
|
231
399
|
/**
|
|
232
400
|
* Materialise deltas into destination tables.
|
|
233
401
|
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
* deletes tombstoned rows. The consumer-owned `props` column is never
|
|
237
|
-
* touched on UPDATE.
|
|
402
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
403
|
+
* BigQuery SQL dialect.
|
|
238
404
|
*/
|
|
239
405
|
async materialise(deltas, schemas) {
|
|
240
|
-
|
|
241
|
-
return Ok(void 0);
|
|
242
|
-
}
|
|
243
|
-
return wrapAsync(async () => {
|
|
244
|
-
const tableRowIds = groupDeltasByTable(deltas);
|
|
245
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
246
|
-
for (const [sourceTable, rowIds] of tableRowIds) {
|
|
247
|
-
const schema = schemaIndex.get(sourceTable);
|
|
248
|
-
if (!schema) continue;
|
|
249
|
-
const pk = resolvePrimaryKey(schema);
|
|
250
|
-
const conflictCols = resolveConflictColumns(schema);
|
|
251
|
-
const soft = isSoftDelete(schema);
|
|
252
|
-
const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
|
|
253
|
-
const deletedAtCol = soft ? `,
|
|
254
|
-
deleted_at TIMESTAMP` : "";
|
|
255
|
-
await this.client.query({
|
|
256
|
-
query: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${schema.table}\` (
|
|
257
|
-
row_id STRING NOT NULL,
|
|
258
|
-
${colDefs},
|
|
259
|
-
props JSON DEFAULT '{}'${deletedAtCol},
|
|
260
|
-
synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
|
|
261
|
-
)
|
|
262
|
-
CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
|
|
263
|
-
location: this.location
|
|
264
|
-
});
|
|
265
|
-
const rowIdArray = [...rowIds];
|
|
266
|
-
const [deltaRows] = await this.client.query({
|
|
267
|
-
query: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
|
|
268
|
-
WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
|
|
269
|
-
ORDER BY hlc ASC`,
|
|
270
|
-
params: { sourceTable, rowIds: rowIdArray },
|
|
271
|
-
location: this.location
|
|
272
|
-
});
|
|
273
|
-
const rowGroups = /* @__PURE__ */ new Map();
|
|
274
|
-
for (const row of deltaRows) {
|
|
275
|
-
let group = rowGroups.get(row.row_id);
|
|
276
|
-
if (!group) {
|
|
277
|
-
group = [];
|
|
278
|
-
rowGroups.set(row.row_id, group);
|
|
279
|
-
}
|
|
280
|
-
group.push({ columns: row.columns, op: row.op });
|
|
281
|
-
}
|
|
282
|
-
const upserts = [];
|
|
283
|
-
const deleteRowIds = [];
|
|
284
|
-
for (const [rowId, group] of rowGroups) {
|
|
285
|
-
const state = mergeLatestState(group);
|
|
286
|
-
if (state === null) {
|
|
287
|
-
deleteRowIds.push(rowId);
|
|
288
|
-
} else {
|
|
289
|
-
upserts.push({ rowId, state });
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
if (upserts.length > 0) {
|
|
293
|
-
const params = {};
|
|
294
|
-
const selects = [];
|
|
295
|
-
for (let i = 0; i < upserts.length; i++) {
|
|
296
|
-
const u = upserts[i];
|
|
297
|
-
params[`rid_${i}`] = u.rowId;
|
|
298
|
-
for (const col of schema.columns) {
|
|
299
|
-
params[`c${schema.columns.indexOf(col)}_${i}`] = u.state[col.name] ?? null;
|
|
300
|
-
}
|
|
301
|
-
const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
|
|
302
|
-
const deletedAtSelect = soft ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
|
|
303
|
-
selects.push(
|
|
304
|
-
`SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
|
|
305
|
-
);
|
|
306
|
-
}
|
|
307
|
-
const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
|
|
308
|
-
const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
|
|
309
|
-
const softUpdateExtra = soft ? ", deleted_at = s.deleted_at" : "";
|
|
310
|
-
const insertColsList = [
|
|
311
|
-
"row_id",
|
|
312
|
-
...schema.columns.map((c) => c.name),
|
|
313
|
-
"props",
|
|
314
|
-
...soft ? ["deleted_at"] : [],
|
|
315
|
-
"synced_at"
|
|
316
|
-
].join(", ");
|
|
317
|
-
const insertValsList = [
|
|
318
|
-
"s.row_id",
|
|
319
|
-
...schema.columns.map((c) => `s.${c.name}`),
|
|
320
|
-
"'{}'",
|
|
321
|
-
...soft ? ["s.deleted_at"] : [],
|
|
322
|
-
"s.synced_at"
|
|
323
|
-
].join(", ");
|
|
324
|
-
const mergeSql = `MERGE \`${this.dataset}.${schema.table}\` AS t
|
|
325
|
-
USING (${selects.join(" UNION ALL ")}) AS s
|
|
326
|
-
ON ${mergeOn}
|
|
327
|
-
WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
|
|
328
|
-
WHEN NOT MATCHED THEN INSERT (${insertColsList})
|
|
329
|
-
VALUES (${insertValsList})`;
|
|
330
|
-
await this.client.query({
|
|
331
|
-
query: mergeSql,
|
|
332
|
-
params,
|
|
333
|
-
location: this.location
|
|
334
|
-
});
|
|
335
|
-
}
|
|
336
|
-
if (deleteRowIds.length > 0) {
|
|
337
|
-
if (soft) {
|
|
338
|
-
await this.client.query({
|
|
339
|
-
query: `UPDATE \`${this.dataset}.${schema.table}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
|
|
340
|
-
params: { rowIds: deleteRowIds },
|
|
341
|
-
location: this.location
|
|
342
|
-
});
|
|
343
|
-
} else {
|
|
344
|
-
await this.client.query({
|
|
345
|
-
query: `DELETE FROM \`${this.dataset}.${schema.table}\` WHERE row_id IN UNNEST(@rowIds)`,
|
|
346
|
-
params: { rowIds: deleteRowIds },
|
|
347
|
-
location: this.location
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
}, "Failed to materialise deltas");
|
|
406
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
353
407
|
}
|
|
354
408
|
/**
|
|
355
409
|
* No-op — BigQuery client is HTTP-based with no persistent connections.
|
|
@@ -460,11 +514,79 @@ var MYSQL_TYPE_MAP = {
|
|
|
460
514
|
function lakeSyncTypeToMySQL(type) {
|
|
461
515
|
return MYSQL_TYPE_MAP[type];
|
|
462
516
|
}
|
|
517
|
+
var MySqlDialect = class {
|
|
518
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
519
|
+
const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
|
|
520
|
+
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
|
|
521
|
+
const deletedAtCol = softDelete ? `, deleted_at TIMESTAMP NULL` : "";
|
|
522
|
+
const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
|
|
523
|
+
return {
|
|
524
|
+
sql: `CREATE TABLE IF NOT EXISTS \`${dest}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`,
|
|
525
|
+
params: []
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
529
|
+
const placeholders = rowIds.map(() => "?").join(", ");
|
|
530
|
+
return {
|
|
531
|
+
sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
|
|
532
|
+
params: [sourceTable, ...rowIds]
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
buildUpsert(dest, schema, _conflictCols, softDelete, upserts) {
|
|
536
|
+
const cols = schema.columns.map((c) => c.name);
|
|
537
|
+
const valuePlaceholders = softDelete ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
|
|
538
|
+
const values = [];
|
|
539
|
+
for (const { rowId, state } of upserts) {
|
|
540
|
+
values.push(rowId);
|
|
541
|
+
for (const col of cols) {
|
|
542
|
+
values.push(state[col] ?? null);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
|
|
546
|
+
const softUpdateExtra = softDelete ? ", deleted_at = NULL" : "";
|
|
547
|
+
const colList = softDelete ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
|
|
548
|
+
return {
|
|
549
|
+
sql: `INSERT INTO \`${dest}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
|
|
550
|
+
params: values
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
554
|
+
const placeholders = deleteIds.map(() => "?").join(", ");
|
|
555
|
+
if (softDelete) {
|
|
556
|
+
return {
|
|
557
|
+
sql: `UPDATE \`${dest}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${placeholders})`,
|
|
558
|
+
params: deleteIds
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
return {
|
|
562
|
+
sql: `DELETE FROM \`${dest}\` WHERE row_id IN (${placeholders})`,
|
|
563
|
+
params: deleteIds
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
};
|
|
463
567
|
var MySQLAdapter = class {
|
|
464
568
|
/** @internal */
|
|
465
569
|
pool;
|
|
570
|
+
dialect = new MySqlDialect();
|
|
466
571
|
constructor(config) {
|
|
467
|
-
this.pool = mysql.createPool(
|
|
572
|
+
this.pool = mysql.createPool({
|
|
573
|
+
uri: config.connectionString,
|
|
574
|
+
connectionLimit: config.poolMax ?? 10,
|
|
575
|
+
connectTimeout: config.connectionTimeoutMs ?? 3e4,
|
|
576
|
+
idleTimeout: config.idleTimeoutMs ?? 1e4
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
get executor() {
|
|
580
|
+
const pool = this.pool;
|
|
581
|
+
return {
|
|
582
|
+
async query(sql, params) {
|
|
583
|
+
await pool.execute(sql, params);
|
|
584
|
+
},
|
|
585
|
+
async queryRows(sql, params) {
|
|
586
|
+
const [rows] = await pool.execute(sql, params);
|
|
587
|
+
return rows;
|
|
588
|
+
}
|
|
589
|
+
};
|
|
468
590
|
}
|
|
469
591
|
/**
|
|
470
592
|
* Insert deltas into the database in a single batch.
|
|
@@ -547,89 +669,11 @@ var MySQLAdapter = class {
|
|
|
547
669
|
/**
|
|
548
670
|
* Materialise deltas into destination tables.
|
|
549
671
|
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
* rows are soft-deleted (default) or hard-deleted. The `props` column
|
|
553
|
-
* is never touched.
|
|
672
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
673
|
+
* MySQL SQL dialect.
|
|
554
674
|
*/
|
|
555
675
|
async materialise(deltas, schemas) {
|
|
556
|
-
|
|
557
|
-
return Ok(void 0);
|
|
558
|
-
}
|
|
559
|
-
return wrapAsync(async () => {
|
|
560
|
-
const grouped = groupDeltasByTable(deltas);
|
|
561
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
562
|
-
for (const [tableName, rowIds] of grouped) {
|
|
563
|
-
const schema = schemaIndex.get(tableName);
|
|
564
|
-
if (!schema) continue;
|
|
565
|
-
const pk = resolvePrimaryKey(schema);
|
|
566
|
-
const soft = isSoftDelete(schema);
|
|
567
|
-
const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
|
|
568
|
-
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
|
|
569
|
-
const deletedAtCol = soft ? `, deleted_at TIMESTAMP NULL` : "";
|
|
570
|
-
const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
|
|
571
|
-
await this.pool.execute(
|
|
572
|
-
`CREATE TABLE IF NOT EXISTS \`${schema.table}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`
|
|
573
|
-
);
|
|
574
|
-
const rowIdArray = [...rowIds];
|
|
575
|
-
const placeholders = rowIdArray.map(() => "?").join(", ");
|
|
576
|
-
const [rows] = await this.pool.execute(
|
|
577
|
-
`SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
|
|
578
|
-
[tableName, ...rowIdArray]
|
|
579
|
-
);
|
|
580
|
-
const byRow = /* @__PURE__ */ new Map();
|
|
581
|
-
for (const row of rows) {
|
|
582
|
-
let list = byRow.get(row.row_id);
|
|
583
|
-
if (!list) {
|
|
584
|
-
list = [];
|
|
585
|
-
byRow.set(row.row_id, list);
|
|
586
|
-
}
|
|
587
|
-
list.push(row);
|
|
588
|
-
}
|
|
589
|
-
const upserts = [];
|
|
590
|
-
const deleteIds = [];
|
|
591
|
-
for (const [rowId, rowDeltas] of byRow) {
|
|
592
|
-
const state = mergeLatestState(rowDeltas);
|
|
593
|
-
if (state === null) {
|
|
594
|
-
deleteIds.push(rowId);
|
|
595
|
-
} else {
|
|
596
|
-
upserts.push({ rowId, state });
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
if (upserts.length > 0) {
|
|
600
|
-
const cols = schema.columns.map((c) => c.name);
|
|
601
|
-
const valuePlaceholders = soft ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
|
|
602
|
-
const values = [];
|
|
603
|
-
for (const { rowId, state } of upserts) {
|
|
604
|
-
values.push(rowId);
|
|
605
|
-
for (const col of cols) {
|
|
606
|
-
values.push(state[col] ?? null);
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
|
|
610
|
-
const softUpdateExtra = soft ? ", deleted_at = NULL" : "";
|
|
611
|
-
const colList = soft ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
|
|
612
|
-
await this.pool.execute(
|
|
613
|
-
`INSERT INTO \`${schema.table}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
|
|
614
|
-
values
|
|
615
|
-
);
|
|
616
|
-
}
|
|
617
|
-
if (deleteIds.length > 0) {
|
|
618
|
-
const delPlaceholders = deleteIds.map(() => "?").join(", ");
|
|
619
|
-
if (soft) {
|
|
620
|
-
await this.pool.execute(
|
|
621
|
-
`UPDATE \`${schema.table}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${delPlaceholders})`,
|
|
622
|
-
deleteIds
|
|
623
|
-
);
|
|
624
|
-
} else {
|
|
625
|
-
await this.pool.execute(
|
|
626
|
-
`DELETE FROM \`${schema.table}\` WHERE row_id IN (${delPlaceholders})`,
|
|
627
|
-
deleteIds
|
|
628
|
-
);
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
}, "Failed to materialise deltas");
|
|
676
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
633
677
|
}
|
|
634
678
|
/** Close the database connection pool and release resources. */
|
|
635
679
|
async close() {
|
|
@@ -657,15 +701,98 @@ var POSTGRES_TYPE_MAP = {
|
|
|
657
701
|
json: "JSONB",
|
|
658
702
|
null: "TEXT"
|
|
659
703
|
};
|
|
704
|
+
var PostgresSqlDialect = class {
|
|
705
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
706
|
+
const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
|
|
707
|
+
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
|
|
708
|
+
const deletedAtCol = softDelete ? `,
|
|
709
|
+
deleted_at TIMESTAMPTZ` : "";
|
|
710
|
+
const uniqueConstraint = schema.externalIdColumn ? `,
|
|
711
|
+
UNIQUE ("${schema.externalIdColumn}")` : "";
|
|
712
|
+
return {
|
|
713
|
+
sql: `CREATE TABLE IF NOT EXISTS "${dest}" (
|
|
714
|
+
row_id TEXT NOT NULL,
|
|
715
|
+
${columnDefs},
|
|
716
|
+
props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
|
|
717
|
+
synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
718
|
+
${pkConstraint}${uniqueConstraint}
|
|
719
|
+
)`,
|
|
720
|
+
params: []
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
724
|
+
return {
|
|
725
|
+
sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
|
|
726
|
+
params: [sourceTable, rowIds]
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
|
|
730
|
+
const colNames = schema.columns.map((c) => c.name);
|
|
731
|
+
const baseCols = ["row_id", ...colNames];
|
|
732
|
+
const allCols = softDelete ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
|
|
733
|
+
const colList = allCols.map((c) => `"${c}"`).join(", ");
|
|
734
|
+
const values = [];
|
|
735
|
+
const valueRows = [];
|
|
736
|
+
const paramsPerRow = allCols.length;
|
|
737
|
+
for (let i = 0; i < upserts.length; i++) {
|
|
738
|
+
const u = upserts[i];
|
|
739
|
+
const offset = i * paramsPerRow;
|
|
740
|
+
const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
|
|
741
|
+
valueRows.push(`(${placeholders.join(", ")})`);
|
|
742
|
+
values.push(u.rowId);
|
|
743
|
+
for (const col of colNames) {
|
|
744
|
+
values.push(u.state[col] ?? null);
|
|
745
|
+
}
|
|
746
|
+
if (softDelete) values.push(null);
|
|
747
|
+
values.push(/* @__PURE__ */ new Date());
|
|
748
|
+
}
|
|
749
|
+
const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
|
|
750
|
+
const updateCols = softDelete ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
|
|
751
|
+
const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
|
|
752
|
+
return {
|
|
753
|
+
sql: `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
|
|
754
|
+
params: values
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
758
|
+
if (softDelete) {
|
|
759
|
+
return {
|
|
760
|
+
sql: `UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
|
|
761
|
+
params: [deleteIds]
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
return {
|
|
765
|
+
sql: `DELETE FROM "${dest}" WHERE row_id = ANY($1)`,
|
|
766
|
+
params: [deleteIds]
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
};
|
|
660
770
|
var PostgresAdapter = class {
|
|
661
771
|
/** @internal */
|
|
662
772
|
pool;
|
|
773
|
+
dialect = new PostgresSqlDialect();
|
|
663
774
|
constructor(config) {
|
|
664
775
|
const poolConfig = {
|
|
665
|
-
connectionString: config.connectionString
|
|
776
|
+
connectionString: config.connectionString,
|
|
777
|
+
max: config.poolMax ?? 10,
|
|
778
|
+
idleTimeoutMillis: config.idleTimeoutMs ?? 1e4,
|
|
779
|
+
connectionTimeoutMillis: config.connectionTimeoutMs ?? 3e4,
|
|
780
|
+
statement_timeout: config.statementTimeoutMs ?? 3e4
|
|
666
781
|
};
|
|
667
782
|
this.pool = new Pool(poolConfig);
|
|
668
783
|
}
|
|
784
|
+
get executor() {
|
|
785
|
+
const pool = this.pool;
|
|
786
|
+
return {
|
|
787
|
+
async query(sql, params) {
|
|
788
|
+
await pool.query(sql, params);
|
|
789
|
+
},
|
|
790
|
+
async queryRows(sql, params) {
|
|
791
|
+
const result = await pool.query(sql, params);
|
|
792
|
+
return result.rows;
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
}
|
|
669
796
|
/**
|
|
670
797
|
* Insert deltas into the database in a single batch.
|
|
671
798
|
* Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
|
|
@@ -762,105 +889,11 @@ CREATE INDEX IF NOT EXISTS idx_lakesync_deltas_table_row ON lakesync_deltas ("ta
|
|
|
762
889
|
/**
|
|
763
890
|
* Materialise deltas into destination tables.
|
|
764
891
|
*
|
|
765
|
-
*
|
|
766
|
-
*
|
|
767
|
-
* rows are deleted. The `props` column is never touched.
|
|
892
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
893
|
+
* Postgres SQL dialect.
|
|
768
894
|
*/
|
|
769
895
|
async materialise(deltas, schemas) {
|
|
770
|
-
|
|
771
|
-
return Ok(void 0);
|
|
772
|
-
}
|
|
773
|
-
return wrapAsync(async () => {
|
|
774
|
-
const grouped = groupDeltasByTable(deltas);
|
|
775
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
776
|
-
for (const [tableName, rowIds] of grouped) {
|
|
777
|
-
const schema = schemaIndex.get(tableName);
|
|
778
|
-
if (!schema) continue;
|
|
779
|
-
const dest = schema.table;
|
|
780
|
-
const pk = resolvePrimaryKey(schema);
|
|
781
|
-
const conflictCols = resolveConflictColumns(schema);
|
|
782
|
-
const soft = isSoftDelete(schema);
|
|
783
|
-
const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
|
|
784
|
-
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
|
|
785
|
-
const deletedAtCol = soft ? `,
|
|
786
|
-
deleted_at TIMESTAMPTZ` : "";
|
|
787
|
-
const uniqueConstraint = schema.externalIdColumn ? `,
|
|
788
|
-
UNIQUE ("${schema.externalIdColumn}")` : "";
|
|
789
|
-
await this.pool.query(
|
|
790
|
-
`CREATE TABLE IF NOT EXISTS "${dest}" (
|
|
791
|
-
row_id TEXT NOT NULL,
|
|
792
|
-
${columnDefs},
|
|
793
|
-
props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
|
|
794
|
-
synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
795
|
-
${pkConstraint}${uniqueConstraint}
|
|
796
|
-
)`
|
|
797
|
-
);
|
|
798
|
-
const sourceTable = schema.sourceTable ?? schema.table;
|
|
799
|
-
const rowIdArray = [...rowIds];
|
|
800
|
-
const deltaResult = await this.pool.query(
|
|
801
|
-
`SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
|
|
802
|
-
[sourceTable, rowIdArray]
|
|
803
|
-
);
|
|
804
|
-
const byRowId = /* @__PURE__ */ new Map();
|
|
805
|
-
for (const row of deltaResult.rows) {
|
|
806
|
-
const rid = row.row_id;
|
|
807
|
-
let arr = byRowId.get(rid);
|
|
808
|
-
if (!arr) {
|
|
809
|
-
arr = [];
|
|
810
|
-
byRowId.set(rid, arr);
|
|
811
|
-
}
|
|
812
|
-
arr.push(row);
|
|
813
|
-
}
|
|
814
|
-
const upserts = [];
|
|
815
|
-
const deleteIds = [];
|
|
816
|
-
for (const [rowId, rows] of byRowId) {
|
|
817
|
-
const state = mergeLatestState(rows);
|
|
818
|
-
if (state !== null) {
|
|
819
|
-
upserts.push({ rowId, state });
|
|
820
|
-
} else {
|
|
821
|
-
deleteIds.push(rowId);
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
if (upserts.length > 0) {
|
|
825
|
-
const colNames = schema.columns.map((c) => c.name);
|
|
826
|
-
const baseCols = ["row_id", ...colNames];
|
|
827
|
-
const allCols = soft ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
|
|
828
|
-
const colList = allCols.map((c) => `"${c}"`).join(", ");
|
|
829
|
-
const values = [];
|
|
830
|
-
const valueRows = [];
|
|
831
|
-
const paramsPerRow = allCols.length;
|
|
832
|
-
for (let i = 0; i < upserts.length; i++) {
|
|
833
|
-
const u = upserts[i];
|
|
834
|
-
const offset = i * paramsPerRow;
|
|
835
|
-
const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
|
|
836
|
-
valueRows.push(`(${placeholders.join(", ")})`);
|
|
837
|
-
values.push(u.rowId);
|
|
838
|
-
for (const col of colNames) {
|
|
839
|
-
values.push(u.state[col] ?? null);
|
|
840
|
-
}
|
|
841
|
-
if (soft) values.push(null);
|
|
842
|
-
values.push(/* @__PURE__ */ new Date());
|
|
843
|
-
}
|
|
844
|
-
const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
|
|
845
|
-
const updateCols = soft ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
|
|
846
|
-
const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
|
|
847
|
-
await this.pool.query(
|
|
848
|
-
`INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
|
|
849
|
-
values
|
|
850
|
-
);
|
|
851
|
-
}
|
|
852
|
-
if (deleteIds.length > 0) {
|
|
853
|
-
if (soft) {
|
|
854
|
-
await this.pool.query(
|
|
855
|
-
`UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
|
|
856
|
-
[deleteIds]
|
|
857
|
-
);
|
|
858
|
-
} else {
|
|
859
|
-
await this.pool.query(`DELETE FROM "${dest}" WHERE row_id = ANY($1)`, [deleteIds]);
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
}, "Failed to materialise deltas");
|
|
896
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
864
897
|
}
|
|
865
898
|
/** Close the database connection pool and release resources. */
|
|
866
899
|
async close() {
|
|
@@ -881,45 +914,46 @@ function rowToRowDelta2(row) {
|
|
|
881
914
|
}
|
|
882
915
|
|
|
883
916
|
// ../adapter/src/factory.ts
|
|
884
|
-
function
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
}
|
|
897
|
-
case "mysql": {
|
|
898
|
-
if (!config.mysql) {
|
|
899
|
-
return Err(new AdapterError("MySQL connector config missing mysql field"));
|
|
900
|
-
}
|
|
901
|
-
return Ok(
|
|
902
|
-
new MySQLAdapter({
|
|
903
|
-
connectionString: config.mysql.connectionString
|
|
904
|
-
})
|
|
905
|
-
);
|
|
906
|
-
}
|
|
907
|
-
case "bigquery": {
|
|
908
|
-
if (!config.bigquery) {
|
|
909
|
-
return Err(new AdapterError("BigQuery connector config missing bigquery field"));
|
|
910
|
-
}
|
|
911
|
-
return Ok(
|
|
912
|
-
new BigQueryAdapter({
|
|
913
|
-
projectId: config.bigquery.projectId,
|
|
914
|
-
dataset: config.bigquery.dataset,
|
|
915
|
-
keyFilename: config.bigquery.keyFilename,
|
|
916
|
-
location: config.bigquery.location
|
|
917
|
-
})
|
|
918
|
-
);
|
|
919
|
-
}
|
|
920
|
-
default:
|
|
921
|
-
return Err(new AdapterError(`Unsupported connector type: ${config.type}`));
|
|
917
|
+
function createAdapterFactoryRegistry(factories = /* @__PURE__ */ new Map()) {
|
|
918
|
+
return buildAdapterFactoryRegistry(new Map(factories));
|
|
919
|
+
}
|
|
920
|
+
function buildAdapterFactoryRegistry(map) {
|
|
921
|
+
return {
|
|
922
|
+
get(type) {
|
|
923
|
+
return map.get(type);
|
|
924
|
+
},
|
|
925
|
+
with(type, factory) {
|
|
926
|
+
const next = new Map(map);
|
|
927
|
+
next.set(type, factory);
|
|
928
|
+
return buildAdapterFactoryRegistry(next);
|
|
922
929
|
}
|
|
930
|
+
};
|
|
931
|
+
}
|
|
932
|
+
function defaultAdapterFactoryRegistry() {
|
|
933
|
+
return createAdapterFactoryRegistry().with("postgres", (c) => {
|
|
934
|
+
const pg = c.postgres;
|
|
935
|
+
return new PostgresAdapter({ connectionString: pg.connectionString });
|
|
936
|
+
}).with("mysql", (c) => {
|
|
937
|
+
const my = c.mysql;
|
|
938
|
+
return new MySQLAdapter({ connectionString: my.connectionString });
|
|
939
|
+
}).with("bigquery", (c) => {
|
|
940
|
+
const bq = c.bigquery;
|
|
941
|
+
return new BigQueryAdapter({
|
|
942
|
+
projectId: bq.projectId,
|
|
943
|
+
dataset: bq.dataset,
|
|
944
|
+
keyFilename: bq.keyFilename,
|
|
945
|
+
location: bq.location
|
|
946
|
+
});
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
function createDatabaseAdapter(config, registry) {
|
|
950
|
+
const reg = registry ?? defaultAdapterFactoryRegistry();
|
|
951
|
+
const factory = reg.get(config.type);
|
|
952
|
+
if (!factory) {
|
|
953
|
+
return Err(new AdapterError(`No adapter factory for connector type "${config.type}"`));
|
|
954
|
+
}
|
|
955
|
+
try {
|
|
956
|
+
return Ok(factory(config));
|
|
923
957
|
} catch (err) {
|
|
924
958
|
return Err(new AdapterError(`Failed to create adapter: ${toError(err).message}`));
|
|
925
959
|
}
|
|
@@ -1214,18 +1248,18 @@ var MinIOAdapter = class {
|
|
|
1214
1248
|
async function createQueryFn(config) {
|
|
1215
1249
|
switch (config.type) {
|
|
1216
1250
|
case "postgres": {
|
|
1217
|
-
|
|
1251
|
+
const pg = config;
|
|
1218
1252
|
const { Pool: Pool2 } = await import("pg");
|
|
1219
|
-
const pool = new Pool2({ connectionString:
|
|
1253
|
+
const pool = new Pool2({ connectionString: pg.postgres.connectionString });
|
|
1220
1254
|
return async (sql, params) => {
|
|
1221
1255
|
const result = await pool.query(sql, params);
|
|
1222
1256
|
return result.rows;
|
|
1223
1257
|
};
|
|
1224
1258
|
}
|
|
1225
1259
|
case "mysql": {
|
|
1226
|
-
|
|
1260
|
+
const my = config;
|
|
1227
1261
|
const mysql2 = await import("mysql2/promise");
|
|
1228
|
-
const pool = mysql2.createPool(
|
|
1262
|
+
const pool = mysql2.createPool(my.mysql.connectionString);
|
|
1229
1263
|
return async (sql, params) => {
|
|
1230
1264
|
const [rows] = await pool.query(sql, params);
|
|
1231
1265
|
return rows;
|
|
@@ -1238,20 +1272,25 @@ async function createQueryFn(config) {
|
|
|
1238
1272
|
|
|
1239
1273
|
export {
|
|
1240
1274
|
lakeSyncTypeToBigQuery,
|
|
1241
|
-
|
|
1242
|
-
|
|
1275
|
+
groupAndMerge,
|
|
1276
|
+
toCause,
|
|
1277
|
+
wrapAsync,
|
|
1278
|
+
mergeLatestState,
|
|
1243
1279
|
resolvePrimaryKey,
|
|
1244
1280
|
resolveConflictColumns,
|
|
1245
1281
|
isSoftDelete,
|
|
1246
1282
|
groupDeltasByTable,
|
|
1247
1283
|
buildSchemaIndex,
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
mergeLatestState,
|
|
1284
|
+
executeMaterialise,
|
|
1285
|
+
BigQuerySqlDialect,
|
|
1251
1286
|
BigQueryAdapter,
|
|
1252
1287
|
CompositeAdapter,
|
|
1288
|
+
MySqlDialect,
|
|
1253
1289
|
MySQLAdapter,
|
|
1290
|
+
PostgresSqlDialect,
|
|
1254
1291
|
PostgresAdapter,
|
|
1292
|
+
createAdapterFactoryRegistry,
|
|
1293
|
+
defaultAdapterFactoryRegistry,
|
|
1255
1294
|
createDatabaseAdapter,
|
|
1256
1295
|
FanOutAdapter,
|
|
1257
1296
|
LifecycleAdapter,
|
|
@@ -1260,4 +1299,4 @@ export {
|
|
|
1260
1299
|
MinIOAdapter,
|
|
1261
1300
|
createQueryFn
|
|
1262
1301
|
};
|
|
1263
|
-
//# sourceMappingURL=chunk-
|
|
1302
|
+
//# sourceMappingURL=chunk-C4KD6YKP.js.map
|