lakesync 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.d.ts +199 -19
- package/dist/adapter.js +19 -3
- package/dist/analyst.js +2 -2
- package/dist/{base-poller-CBvhdvcj.d.ts → base-poller-Bj9kX9dv.d.ts} +76 -19
- package/dist/catalogue.d.ts +1 -1
- package/dist/catalogue.js +3 -3
- package/dist/chunk-DGUM43GV.js +11 -0
- package/dist/{chunk-PWGQ3PXE.js → chunk-JI4C4R5H.js} +280 -140
- package/dist/chunk-JI4C4R5H.js.map +1 -0
- package/dist/{chunk-L4ZL5JA7.js → chunk-KVSWLIJR.js} +2 -2
- package/dist/{chunk-7UBS6MFH.js → chunk-LDFFCG2K.js} +377 -247
- package/dist/chunk-LDFFCG2K.js.map +1 -0
- package/dist/{chunk-Z7FGLEQU.js → chunk-LPWXOYNS.js} +376 -287
- package/dist/chunk-LPWXOYNS.js.map +1 -0
- package/dist/{chunk-SZSGSTVZ.js → chunk-PYRS74YP.js} +15 -4
- package/dist/{chunk-SZSGSTVZ.js.map → chunk-PYRS74YP.js.map} +1 -1
- package/dist/{chunk-TVLTXHW6.js → chunk-QNITY4F6.js} +30 -7
- package/dist/{chunk-TVLTXHW6.js.map → chunk-QNITY4F6.js.map} +1 -1
- package/dist/{chunk-46CKACNC.js → chunk-SSICS5KI.js} +2 -2
- package/dist/{chunk-B3QEUG6E.js → chunk-TMLG32QV.js} +2 -2
- package/dist/client.d.ts +164 -13
- package/dist/client.js +310 -163
- package/dist/client.js.map +1 -1
- package/dist/compactor.d.ts +1 -1
- package/dist/compactor.js +4 -4
- package/dist/connector-jira.d.ts +2 -2
- package/dist/connector-jira.js +3 -3
- package/dist/connector-salesforce.d.ts +2 -2
- package/dist/connector-salesforce.js +3 -3
- package/dist/{coordinator-DN8D8C7W.d.ts → coordinator-NXy6tA0h.d.ts} +23 -16
- package/dist/{db-types-B6_JKQWK.d.ts → db-types-CfLMUBfW.d.ts} +1 -1
- package/dist/gateway-server.d.ts +158 -64
- package/dist/gateway-server.js +482 -4003
- package/dist/gateway-server.js.map +1 -1
- package/dist/gateway.d.ts +61 -104
- package/dist/gateway.js +12 -6
- package/dist/index.d.ts +45 -10
- package/dist/index.js +14 -2
- package/dist/parquet.d.ts +1 -1
- package/dist/parquet.js +3 -3
- package/dist/proto.d.ts +1 -1
- package/dist/proto.js +3 -3
- package/dist/react.d.ts +47 -10
- package/dist/react.js +88 -40
- package/dist/react.js.map +1 -1
- package/dist/{registry-BN_9spxE.d.ts → registry-BcspAtZI.d.ts} +19 -4
- package/dist/{gateway-CvO7Xy3T.d.ts → request-handler-pUvL7ozF.d.ts} +139 -10
- package/dist/{resolver-BZURzdlL.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
- package/dist/{src-RR7I76OL.js → src-B6NLV3FP.js} +4 -4
- package/dist/{src-SLVE5567.js → src-ROW4XLO7.js} +15 -3
- package/dist/{src-V2CTPR7V.js → src-ZRHKG42A.js} +4 -4
- package/dist/{types-GGBfZBKQ.d.ts → types-BdGBv2ba.d.ts} +23 -2
- package/dist/{types-D-E0VrfS.d.ts → types-BrcD1oJg.d.ts} +26 -19
- package/package.json +1 -1
- package/dist/chunk-7D4SUZUM.js +0 -38
- package/dist/chunk-7UBS6MFH.js.map +0 -1
- package/dist/chunk-PWGQ3PXE.js.map +0 -1
- package/dist/chunk-Z7FGLEQU.js.map +0 -1
- /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
- /package/dist/{chunk-L4ZL5JA7.js.map → chunk-KVSWLIJR.js.map} +0 -0
- /package/dist/{chunk-46CKACNC.js.map → chunk-SSICS5KI.js.map} +0 -0
- /package/dist/{chunk-B3QEUG6E.js.map → chunk-TMLG32QV.js.map} +0 -0
- /package/dist/{src-RR7I76OL.js.map → src-B6NLV3FP.js.map} +0 -0
- /package/dist/{src-SLVE5567.js.map → src-ROW4XLO7.js.map} +0 -0
- /package/dist/{src-V2CTPR7V.js.map → src-ZRHKG42A.js.map} +0 -0
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
Err,
|
|
4
4
|
Ok,
|
|
5
5
|
toError
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-LDFFCG2K.js";
|
|
7
7
|
|
|
8
8
|
// ../adapter/src/db-types.ts
|
|
9
9
|
var BIGQUERY_TYPE_MAP = {
|
|
@@ -20,32 +20,29 @@ function isDatabaseAdapter(adapter) {
|
|
|
20
20
|
return adapter !== null && typeof adapter === "object" && "insertDeltas" in adapter && "queryDeltasSince" in adapter && typeof adapter.insertDeltas === "function";
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
// ../adapter/src/
|
|
24
|
-
function
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (!rowIds) {
|
|
32
|
-
rowIds = /* @__PURE__ */ new Set();
|
|
33
|
-
result.set(delta.table, rowIds);
|
|
23
|
+
// ../adapter/src/shared.ts
|
|
24
|
+
function groupAndMerge(rows) {
|
|
25
|
+
const byRowId = /* @__PURE__ */ new Map();
|
|
26
|
+
for (const row of rows) {
|
|
27
|
+
let arr = byRowId.get(row.row_id);
|
|
28
|
+
if (!arr) {
|
|
29
|
+
arr = [];
|
|
30
|
+
byRowId.set(row.row_id, arr);
|
|
34
31
|
}
|
|
35
|
-
|
|
32
|
+
arr.push(row);
|
|
36
33
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
34
|
+
const upserts = [];
|
|
35
|
+
const deleteIds = [];
|
|
36
|
+
for (const [rowId, group] of byRowId) {
|
|
37
|
+
const state = mergeLatestState(group);
|
|
38
|
+
if (state !== null) {
|
|
39
|
+
upserts.push({ rowId, state });
|
|
40
|
+
} else {
|
|
41
|
+
deleteIds.push(rowId);
|
|
42
|
+
}
|
|
44
43
|
}
|
|
45
|
-
return
|
|
44
|
+
return { upserts, deleteIds };
|
|
46
45
|
}
|
|
47
|
-
|
|
48
|
-
// ../adapter/src/shared.ts
|
|
49
46
|
function toCause(error) {
|
|
50
47
|
return error instanceof Error ? error : void 0;
|
|
51
48
|
}
|
|
@@ -80,6 +77,72 @@ function mergeLatestState(rows) {
|
|
|
80
77
|
return state;
|
|
81
78
|
}
|
|
82
79
|
|
|
80
|
+
// ../adapter/src/materialise.ts
|
|
81
|
+
function isMaterialisable(adapter) {
|
|
82
|
+
return adapter !== null && typeof adapter === "object" && "materialise" in adapter && typeof adapter.materialise === "function";
|
|
83
|
+
}
|
|
84
|
+
function resolvePrimaryKey(schema) {
|
|
85
|
+
return schema.primaryKey ?? ["row_id"];
|
|
86
|
+
}
|
|
87
|
+
function resolveConflictColumns(schema) {
|
|
88
|
+
return schema.externalIdColumn ? [schema.externalIdColumn] : resolvePrimaryKey(schema);
|
|
89
|
+
}
|
|
90
|
+
function isSoftDelete(schema) {
|
|
91
|
+
return schema.softDelete !== false;
|
|
92
|
+
}
|
|
93
|
+
function groupDeltasByTable(deltas) {
|
|
94
|
+
const result = /* @__PURE__ */ new Map();
|
|
95
|
+
for (const delta of deltas) {
|
|
96
|
+
let rowIds = result.get(delta.table);
|
|
97
|
+
if (!rowIds) {
|
|
98
|
+
rowIds = /* @__PURE__ */ new Set();
|
|
99
|
+
result.set(delta.table, rowIds);
|
|
100
|
+
}
|
|
101
|
+
rowIds.add(delta.rowId);
|
|
102
|
+
}
|
|
103
|
+
return result;
|
|
104
|
+
}
|
|
105
|
+
function buildSchemaIndex(schemas) {
|
|
106
|
+
const index = /* @__PURE__ */ new Map();
|
|
107
|
+
for (const schema of schemas) {
|
|
108
|
+
const key = schema.sourceTable ?? schema.table;
|
|
109
|
+
index.set(key, schema);
|
|
110
|
+
}
|
|
111
|
+
return index;
|
|
112
|
+
}
|
|
113
|
+
async function executeMaterialise(executor, dialect, deltas, schemas) {
|
|
114
|
+
if (deltas.length === 0) {
|
|
115
|
+
return Ok(void 0);
|
|
116
|
+
}
|
|
117
|
+
return wrapAsync(async () => {
|
|
118
|
+
const grouped = groupDeltasByTable(deltas);
|
|
119
|
+
const schemaIndex = buildSchemaIndex(schemas);
|
|
120
|
+
for (const [tableName, rowIds] of grouped) {
|
|
121
|
+
const schema = schemaIndex.get(tableName);
|
|
122
|
+
if (!schema) continue;
|
|
123
|
+
const dest = schema.table;
|
|
124
|
+
const pk = resolvePrimaryKey(schema);
|
|
125
|
+
const conflictCols = resolveConflictColumns(schema);
|
|
126
|
+
const soft = isSoftDelete(schema);
|
|
127
|
+
const createStmt = dialect.createDestinationTable(dest, schema, pk, soft);
|
|
128
|
+
await executor.query(createStmt.sql, createStmt.params);
|
|
129
|
+
const sourceTable = schema.sourceTable ?? schema.table;
|
|
130
|
+
const rowIdArray = [...rowIds];
|
|
131
|
+
const historyStmt = dialect.queryDeltaHistory(sourceTable, rowIdArray);
|
|
132
|
+
const rows = await executor.queryRows(historyStmt.sql, historyStmt.params);
|
|
133
|
+
const { upserts, deleteIds } = groupAndMerge(rows);
|
|
134
|
+
if (upserts.length > 0) {
|
|
135
|
+
const upsertStmt = dialect.buildUpsert(dest, schema, conflictCols, soft, upserts);
|
|
136
|
+
await executor.query(upsertStmt.sql, upsertStmt.params);
|
|
137
|
+
}
|
|
138
|
+
if (deleteIds.length > 0) {
|
|
139
|
+
const deleteStmt = dialect.buildDelete(dest, deleteIds, soft);
|
|
140
|
+
await executor.query(deleteStmt.sql, deleteStmt.params);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}, "Failed to materialise deltas");
|
|
144
|
+
}
|
|
145
|
+
|
|
83
146
|
// ../adapter/src/bigquery.ts
|
|
84
147
|
import { BigQuery } from "@google-cloud/bigquery";
|
|
85
148
|
function rowToRowDelta(row) {
|
|
@@ -96,6 +159,121 @@ function rowToRowDelta(row) {
|
|
|
96
159
|
op: row.op
|
|
97
160
|
};
|
|
98
161
|
}
|
|
162
|
+
var BigQuerySqlDialect = class {
|
|
163
|
+
constructor(dataset) {
|
|
164
|
+
this.dataset = dataset;
|
|
165
|
+
}
|
|
166
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
167
|
+
const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
|
|
168
|
+
const deletedAtCol = softDelete ? `,
|
|
169
|
+
deleted_at TIMESTAMP` : "";
|
|
170
|
+
return {
|
|
171
|
+
sql: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${dest}\` (
|
|
172
|
+
row_id STRING NOT NULL,
|
|
173
|
+
${colDefs},
|
|
174
|
+
props JSON DEFAULT '{}'${deletedAtCol},
|
|
175
|
+
synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
|
|
176
|
+
)
|
|
177
|
+
CLUSTER BY ${pk.map((c) => c === "row_id" ? "row_id" : c).join(", ")}`,
|
|
178
|
+
params: []
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
182
|
+
return {
|
|
183
|
+
sql: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
|
|
184
|
+
WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
|
|
185
|
+
ORDER BY hlc ASC`,
|
|
186
|
+
params: [
|
|
187
|
+
["sourceTable", sourceTable],
|
|
188
|
+
["rowIds", rowIds]
|
|
189
|
+
]
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
|
|
193
|
+
const namedParams = [];
|
|
194
|
+
const selects = [];
|
|
195
|
+
for (let i = 0; i < upserts.length; i++) {
|
|
196
|
+
const u = upserts[i];
|
|
197
|
+
namedParams.push([`rid_${i}`, u.rowId]);
|
|
198
|
+
for (const col of schema.columns) {
|
|
199
|
+
namedParams.push([`c${schema.columns.indexOf(col)}_${i}`, u.state[col.name] ?? null]);
|
|
200
|
+
}
|
|
201
|
+
const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
|
|
202
|
+
const deletedAtSelect = softDelete ? ", CAST(NULL AS TIMESTAMP) AS deleted_at" : "";
|
|
203
|
+
selects.push(
|
|
204
|
+
`SELECT @rid_${i} AS row_id, ${colSelects}${deletedAtSelect}, CURRENT_TIMESTAMP() AS synced_at`
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
const mergeOn = conflictCols.map((c) => `t.${c === "row_id" ? "row_id" : c} = s.${c === "row_id" ? "row_id" : c}`).join(" AND ");
|
|
208
|
+
const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
|
|
209
|
+
const softUpdateExtra = softDelete ? ", deleted_at = s.deleted_at" : "";
|
|
210
|
+
const insertColsList = [
|
|
211
|
+
"row_id",
|
|
212
|
+
...schema.columns.map((c) => c.name),
|
|
213
|
+
"props",
|
|
214
|
+
...softDelete ? ["deleted_at"] : [],
|
|
215
|
+
"synced_at"
|
|
216
|
+
].join(", ");
|
|
217
|
+
const insertValsList = [
|
|
218
|
+
"s.row_id",
|
|
219
|
+
...schema.columns.map((c) => `s.${c.name}`),
|
|
220
|
+
"'{}'",
|
|
221
|
+
...softDelete ? ["s.deleted_at"] : [],
|
|
222
|
+
"s.synced_at"
|
|
223
|
+
].join(", ");
|
|
224
|
+
return {
|
|
225
|
+
sql: `MERGE \`${this.dataset}.${dest}\` AS t
|
|
226
|
+
USING (${selects.join(" UNION ALL ")}) AS s
|
|
227
|
+
ON ${mergeOn}
|
|
228
|
+
WHEN MATCHED THEN UPDATE SET ${updateSet}${softUpdateExtra}, synced_at = s.synced_at
|
|
229
|
+
WHEN NOT MATCHED THEN INSERT (${insertColsList})
|
|
230
|
+
VALUES (${insertValsList})`,
|
|
231
|
+
params: namedParams
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
235
|
+
if (softDelete) {
|
|
236
|
+
return {
|
|
237
|
+
sql: `UPDATE \`${this.dataset}.${dest}\` SET deleted_at = CURRENT_TIMESTAMP(), synced_at = CURRENT_TIMESTAMP() WHERE row_id IN UNNEST(@rowIds)`,
|
|
238
|
+
params: [["rowIds", deleteIds]]
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
return {
|
|
242
|
+
sql: `DELETE FROM \`${this.dataset}.${dest}\` WHERE row_id IN UNNEST(@rowIds)`,
|
|
243
|
+
params: [["rowIds", deleteIds]]
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
function createBigQueryExecutor(client, location) {
|
|
248
|
+
function toNamedParams(params) {
|
|
249
|
+
if (params.length === 0) return void 0;
|
|
250
|
+
const result = {};
|
|
251
|
+
for (const entry of params) {
|
|
252
|
+
const [key, value] = entry;
|
|
253
|
+
result[key] = value;
|
|
254
|
+
}
|
|
255
|
+
return result;
|
|
256
|
+
}
|
|
257
|
+
return {
|
|
258
|
+
async query(sql, params) {
|
|
259
|
+
const namedParams = toNamedParams(params);
|
|
260
|
+
await client.query({
|
|
261
|
+
query: sql,
|
|
262
|
+
params: namedParams,
|
|
263
|
+
location
|
|
264
|
+
});
|
|
265
|
+
},
|
|
266
|
+
async queryRows(sql, params) {
|
|
267
|
+
const namedParams = toNamedParams(params);
|
|
268
|
+
const [rows] = await client.query({
|
|
269
|
+
query: sql,
|
|
270
|
+
params: namedParams,
|
|
271
|
+
location
|
|
272
|
+
});
|
|
273
|
+
return rows;
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
}
|
|
99
277
|
var BigQueryAdapter = class {
|
|
100
278
|
/** @internal */
|
|
101
279
|
client;
|
|
@@ -103,6 +281,8 @@ var BigQueryAdapter = class {
|
|
|
103
281
|
dataset;
|
|
104
282
|
/** @internal */
|
|
105
283
|
location;
|
|
284
|
+
dialect;
|
|
285
|
+
executor;
|
|
106
286
|
constructor(config) {
|
|
107
287
|
this.client = new BigQuery({
|
|
108
288
|
projectId: config.projectId,
|
|
@@ -110,6 +290,8 @@ var BigQueryAdapter = class {
|
|
|
110
290
|
});
|
|
111
291
|
this.dataset = config.dataset;
|
|
112
292
|
this.location = config.location ?? "US";
|
|
293
|
+
this.dialect = new BigQuerySqlDialect(this.dataset);
|
|
294
|
+
this.executor = createBigQueryExecutor(this.client, this.location);
|
|
113
295
|
}
|
|
114
296
|
/**
|
|
115
297
|
* Insert deltas into the database in a single batch.
|
|
@@ -222,106 +404,11 @@ CLUSTER BY \`table\`, hlc`,
|
|
|
222
404
|
/**
|
|
223
405
|
* Materialise deltas into destination tables.
|
|
224
406
|
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
* deletes tombstoned rows. The consumer-owned `props` column is never
|
|
228
|
-
* touched on UPDATE.
|
|
407
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
408
|
+
* BigQuery SQL dialect.
|
|
229
409
|
*/
|
|
230
410
|
async materialise(deltas, schemas) {
|
|
231
|
-
|
|
232
|
-
return Ok(void 0);
|
|
233
|
-
}
|
|
234
|
-
return wrapAsync(async () => {
|
|
235
|
-
const tableRowIds = groupDeltasByTable(deltas);
|
|
236
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
237
|
-
for (const [sourceTable, rowIds] of tableRowIds) {
|
|
238
|
-
const schema = schemaIndex.get(sourceTable);
|
|
239
|
-
if (!schema) continue;
|
|
240
|
-
const colDefs = schema.columns.map((c) => `${c.name} ${lakeSyncTypeToBigQuery(c.type)}`).join(", ");
|
|
241
|
-
await this.client.query({
|
|
242
|
-
query: `CREATE TABLE IF NOT EXISTS \`${this.dataset}.${schema.table}\` (
|
|
243
|
-
row_id STRING NOT NULL,
|
|
244
|
-
${colDefs},
|
|
245
|
-
props JSON DEFAULT '{}',
|
|
246
|
-
synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP()
|
|
247
|
-
)`,
|
|
248
|
-
location: this.location
|
|
249
|
-
});
|
|
250
|
-
const rowIdArray = [...rowIds];
|
|
251
|
-
const [deltaRows] = await this.client.query({
|
|
252
|
-
query: `SELECT row_id, columns, op FROM \`${this.dataset}.lakesync_deltas\`
|
|
253
|
-
WHERE \`table\` = @sourceTable AND row_id IN UNNEST(@rowIds)
|
|
254
|
-
ORDER BY hlc ASC`,
|
|
255
|
-
params: { sourceTable, rowIds: rowIdArray },
|
|
256
|
-
location: this.location
|
|
257
|
-
});
|
|
258
|
-
const rowGroups = /* @__PURE__ */ new Map();
|
|
259
|
-
for (const row of deltaRows) {
|
|
260
|
-
let group = rowGroups.get(row.row_id);
|
|
261
|
-
if (!group) {
|
|
262
|
-
group = [];
|
|
263
|
-
rowGroups.set(row.row_id, group);
|
|
264
|
-
}
|
|
265
|
-
group.push({ columns: row.columns, op: row.op });
|
|
266
|
-
}
|
|
267
|
-
const upserts = [];
|
|
268
|
-
const deleteRowIds = [];
|
|
269
|
-
for (const [rowId, group] of rowGroups) {
|
|
270
|
-
const state = mergeLatestState(group);
|
|
271
|
-
if (state === null) {
|
|
272
|
-
deleteRowIds.push(rowId);
|
|
273
|
-
} else {
|
|
274
|
-
upserts.push({ rowId, state });
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
if (upserts.length > 0) {
|
|
278
|
-
const params = {};
|
|
279
|
-
const selects = [];
|
|
280
|
-
for (let i = 0; i < upserts.length; i++) {
|
|
281
|
-
const u = upserts[i];
|
|
282
|
-
params[`rid_${i}`] = u.rowId;
|
|
283
|
-
for (const col of schema.columns) {
|
|
284
|
-
params[`c${schema.columns.indexOf(col)}_${i}`] = u.state[col.name] ?? null;
|
|
285
|
-
}
|
|
286
|
-
const colSelects = schema.columns.map((col, ci) => `@c${ci}_${i} AS ${col.name}`).join(", ");
|
|
287
|
-
selects.push(
|
|
288
|
-
`SELECT @rid_${i} AS row_id, ${colSelects}, CURRENT_TIMESTAMP() AS synced_at`
|
|
289
|
-
);
|
|
290
|
-
}
|
|
291
|
-
const updateSet = schema.columns.map((col) => `${col.name} = s.${col.name}`).join(", ");
|
|
292
|
-
const insertCols = [
|
|
293
|
-
"row_id",
|
|
294
|
-
...schema.columns.map((c) => c.name),
|
|
295
|
-
"props",
|
|
296
|
-
"synced_at"
|
|
297
|
-
].join(", ");
|
|
298
|
-
const insertVals = [
|
|
299
|
-
"s.row_id",
|
|
300
|
-
...schema.columns.map((c) => `s.${c.name}`),
|
|
301
|
-
"'{}'",
|
|
302
|
-
"s.synced_at"
|
|
303
|
-
].join(", ");
|
|
304
|
-
const mergeSql = `MERGE \`${this.dataset}.${schema.table}\` AS t
|
|
305
|
-
USING (${selects.join(" UNION ALL ")}) AS s
|
|
306
|
-
ON t.row_id = s.row_id
|
|
307
|
-
WHEN MATCHED THEN UPDATE SET ${updateSet}, synced_at = s.synced_at
|
|
308
|
-
WHEN NOT MATCHED THEN INSERT (${insertCols})
|
|
309
|
-
VALUES (${insertVals})`;
|
|
310
|
-
await this.client.query({
|
|
311
|
-
query: mergeSql,
|
|
312
|
-
params,
|
|
313
|
-
location: this.location
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
if (deleteRowIds.length > 0) {
|
|
317
|
-
await this.client.query({
|
|
318
|
-
query: `DELETE FROM \`${this.dataset}.${schema.table}\` WHERE row_id IN UNNEST(@rowIds)`,
|
|
319
|
-
params: { rowIds: deleteRowIds },
|
|
320
|
-
location: this.location
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
}, "Failed to materialise deltas");
|
|
411
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
325
412
|
}
|
|
326
413
|
/**
|
|
327
414
|
* No-op — BigQuery client is HTTP-based with no persistent connections.
|
|
@@ -432,12 +519,75 @@ var MYSQL_TYPE_MAP = {
|
|
|
432
519
|
function lakeSyncTypeToMySQL(type) {
|
|
433
520
|
return MYSQL_TYPE_MAP[type];
|
|
434
521
|
}
|
|
522
|
+
var MySqlDialect = class {
|
|
523
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
524
|
+
const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
|
|
525
|
+
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `\`${c}\``).join(", ")})`;
|
|
526
|
+
const deletedAtCol = softDelete ? `, deleted_at TIMESTAMP NULL` : "";
|
|
527
|
+
const uniqueConstraint = schema.externalIdColumn ? `, UNIQUE KEY (\`${schema.externalIdColumn}\`)` : "";
|
|
528
|
+
return {
|
|
529
|
+
sql: `CREATE TABLE IF NOT EXISTS \`${dest}\` (row_id VARCHAR(255) NOT NULL, ${typedCols}, props JSON NOT NULL DEFAULT ('{}')${deletedAtCol}, synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ${pkConstraint}${uniqueConstraint})`,
|
|
530
|
+
params: []
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
534
|
+
const placeholders = rowIds.map(() => "?").join(", ");
|
|
535
|
+
return {
|
|
536
|
+
sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
|
|
537
|
+
params: [sourceTable, ...rowIds]
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
buildUpsert(dest, schema, _conflictCols, softDelete, upserts) {
|
|
541
|
+
const cols = schema.columns.map((c) => c.name);
|
|
542
|
+
const valuePlaceholders = softDelete ? upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NULL, NOW())`).join(", ") : upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
|
|
543
|
+
const values = [];
|
|
544
|
+
for (const { rowId, state } of upserts) {
|
|
545
|
+
values.push(rowId);
|
|
546
|
+
for (const col of cols) {
|
|
547
|
+
values.push(state[col] ?? null);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
|
|
551
|
+
const softUpdateExtra = softDelete ? ", deleted_at = NULL" : "";
|
|
552
|
+
const colList = softDelete ? `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, deleted_at, synced_at` : `row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at`;
|
|
553
|
+
return {
|
|
554
|
+
sql: `INSERT INTO \`${dest}\` (${colList}) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}${softUpdateExtra}, synced_at = VALUES(synced_at)`,
|
|
555
|
+
params: values
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
559
|
+
const placeholders = deleteIds.map(() => "?").join(", ");
|
|
560
|
+
if (softDelete) {
|
|
561
|
+
return {
|
|
562
|
+
sql: `UPDATE \`${dest}\` SET deleted_at = NOW(), synced_at = NOW() WHERE row_id IN (${placeholders})`,
|
|
563
|
+
params: deleteIds
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
return {
|
|
567
|
+
sql: `DELETE FROM \`${dest}\` WHERE row_id IN (${placeholders})`,
|
|
568
|
+
params: deleteIds
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
};
|
|
435
572
|
var MySQLAdapter = class {
|
|
436
573
|
/** @internal */
|
|
437
574
|
pool;
|
|
575
|
+
dialect = new MySqlDialect();
|
|
438
576
|
constructor(config) {
|
|
439
577
|
this.pool = mysql.createPool(config.connectionString);
|
|
440
578
|
}
|
|
579
|
+
get executor() {
|
|
580
|
+
const pool = this.pool;
|
|
581
|
+
return {
|
|
582
|
+
async query(sql, params) {
|
|
583
|
+
await pool.execute(sql, params);
|
|
584
|
+
},
|
|
585
|
+
async queryRows(sql, params) {
|
|
586
|
+
const [rows] = await pool.execute(sql, params);
|
|
587
|
+
return rows;
|
|
588
|
+
}
|
|
589
|
+
};
|
|
590
|
+
}
|
|
441
591
|
/**
|
|
442
592
|
* Insert deltas into the database in a single batch.
|
|
443
593
|
* Uses INSERT IGNORE for idempotent writes — duplicate deltaIds are silently skipped.
|
|
@@ -519,74 +669,11 @@ var MySQLAdapter = class {
|
|
|
519
669
|
/**
|
|
520
670
|
* Materialise deltas into destination tables.
|
|
521
671
|
*
|
|
522
|
-
*
|
|
523
|
-
*
|
|
524
|
-
* rows are deleted. The `props` column is never touched.
|
|
672
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
673
|
+
* MySQL SQL dialect.
|
|
525
674
|
*/
|
|
526
675
|
async materialise(deltas, schemas) {
|
|
527
|
-
|
|
528
|
-
return Ok(void 0);
|
|
529
|
-
}
|
|
530
|
-
return wrapAsync(async () => {
|
|
531
|
-
const grouped = groupDeltasByTable(deltas);
|
|
532
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
533
|
-
for (const [tableName, rowIds] of grouped) {
|
|
534
|
-
const schema = schemaIndex.get(tableName);
|
|
535
|
-
if (!schema) continue;
|
|
536
|
-
const typedCols = schema.columns.map((col) => `\`${col.name}\` ${lakeSyncTypeToMySQL(col.type)}`).join(", ");
|
|
537
|
-
await this.pool.execute(
|
|
538
|
-
`CREATE TABLE IF NOT EXISTS \`${schema.table}\` (row_id VARCHAR(255) PRIMARY KEY, ${typedCols}, props JSON NOT NULL DEFAULT ('{}'), synced_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)`
|
|
539
|
-
);
|
|
540
|
-
const rowIdArray = [...rowIds];
|
|
541
|
-
const placeholders = rowIdArray.map(() => "?").join(", ");
|
|
542
|
-
const [rows] = await this.pool.execute(
|
|
543
|
-
`SELECT row_id, columns, op FROM lakesync_deltas WHERE \`table\` = ? AND row_id IN (${placeholders}) ORDER BY hlc ASC`,
|
|
544
|
-
[tableName, ...rowIdArray]
|
|
545
|
-
);
|
|
546
|
-
const byRow = /* @__PURE__ */ new Map();
|
|
547
|
-
for (const row of rows) {
|
|
548
|
-
let list = byRow.get(row.row_id);
|
|
549
|
-
if (!list) {
|
|
550
|
-
list = [];
|
|
551
|
-
byRow.set(row.row_id, list);
|
|
552
|
-
}
|
|
553
|
-
list.push(row);
|
|
554
|
-
}
|
|
555
|
-
const upserts = [];
|
|
556
|
-
const deleteIds = [];
|
|
557
|
-
for (const [rowId, rowDeltas] of byRow) {
|
|
558
|
-
const state = mergeLatestState(rowDeltas);
|
|
559
|
-
if (state === null) {
|
|
560
|
-
deleteIds.push(rowId);
|
|
561
|
-
} else {
|
|
562
|
-
upserts.push({ rowId, state });
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
if (upserts.length > 0) {
|
|
566
|
-
const cols = schema.columns.map((c) => c.name);
|
|
567
|
-
const valuePlaceholders = upserts.map(() => `(?, ${cols.map(() => "?").join(", ")}, NOW())`).join(", ");
|
|
568
|
-
const values = [];
|
|
569
|
-
for (const { rowId, state } of upserts) {
|
|
570
|
-
values.push(rowId);
|
|
571
|
-
for (const col of cols) {
|
|
572
|
-
values.push(state[col] ?? null);
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
const updateCols = cols.map((c) => `\`${c}\` = VALUES(\`${c}\`)`).join(", ");
|
|
576
|
-
await this.pool.execute(
|
|
577
|
-
`INSERT INTO \`${schema.table}\` (row_id, ${cols.map((c) => `\`${c}\``).join(", ")}, synced_at) VALUES ${valuePlaceholders} ON DUPLICATE KEY UPDATE ${updateCols}, synced_at = VALUES(synced_at)`,
|
|
578
|
-
values
|
|
579
|
-
);
|
|
580
|
-
}
|
|
581
|
-
if (deleteIds.length > 0) {
|
|
582
|
-
const delPlaceholders = deleteIds.map(() => "?").join(", ");
|
|
583
|
-
await this.pool.execute(
|
|
584
|
-
`DELETE FROM \`${schema.table}\` WHERE row_id IN (${delPlaceholders})`,
|
|
585
|
-
deleteIds
|
|
586
|
-
);
|
|
587
|
-
}
|
|
588
|
-
}
|
|
589
|
-
}, "Failed to materialise deltas");
|
|
676
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
590
677
|
}
|
|
591
678
|
/** Close the database connection pool and release resources. */
|
|
592
679
|
async close() {
|
|
@@ -614,15 +701,94 @@ var POSTGRES_TYPE_MAP = {
|
|
|
614
701
|
json: "JSONB",
|
|
615
702
|
null: "TEXT"
|
|
616
703
|
};
|
|
704
|
+
var PostgresSqlDialect = class {
|
|
705
|
+
createDestinationTable(dest, schema, pk, softDelete) {
|
|
706
|
+
const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
|
|
707
|
+
const pkConstraint = `PRIMARY KEY (${pk.map((c) => `"${c}"`).join(", ")})`;
|
|
708
|
+
const deletedAtCol = softDelete ? `,
|
|
709
|
+
deleted_at TIMESTAMPTZ` : "";
|
|
710
|
+
const uniqueConstraint = schema.externalIdColumn ? `,
|
|
711
|
+
UNIQUE ("${schema.externalIdColumn}")` : "";
|
|
712
|
+
return {
|
|
713
|
+
sql: `CREATE TABLE IF NOT EXISTS "${dest}" (
|
|
714
|
+
row_id TEXT NOT NULL,
|
|
715
|
+
${columnDefs},
|
|
716
|
+
props JSONB NOT NULL DEFAULT '{}'${deletedAtCol},
|
|
717
|
+
synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
718
|
+
${pkConstraint}${uniqueConstraint}
|
|
719
|
+
)`,
|
|
720
|
+
params: []
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
queryDeltaHistory(sourceTable, rowIds) {
|
|
724
|
+
return {
|
|
725
|
+
sql: `SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
|
|
726
|
+
params: [sourceTable, rowIds]
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
buildUpsert(dest, schema, conflictCols, softDelete, upserts) {
|
|
730
|
+
const colNames = schema.columns.map((c) => c.name);
|
|
731
|
+
const baseCols = ["row_id", ...colNames];
|
|
732
|
+
const allCols = softDelete ? [...baseCols, "deleted_at", "synced_at"] : [...baseCols, "synced_at"];
|
|
733
|
+
const colList = allCols.map((c) => `"${c}"`).join(", ");
|
|
734
|
+
const values = [];
|
|
735
|
+
const valueRows = [];
|
|
736
|
+
const paramsPerRow = allCols.length;
|
|
737
|
+
for (let i = 0; i < upserts.length; i++) {
|
|
738
|
+
const u = upserts[i];
|
|
739
|
+
const offset = i * paramsPerRow;
|
|
740
|
+
const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
|
|
741
|
+
valueRows.push(`(${placeholders.join(", ")})`);
|
|
742
|
+
values.push(u.rowId);
|
|
743
|
+
for (const col of colNames) {
|
|
744
|
+
values.push(u.state[col] ?? null);
|
|
745
|
+
}
|
|
746
|
+
if (softDelete) values.push(null);
|
|
747
|
+
values.push(/* @__PURE__ */ new Date());
|
|
748
|
+
}
|
|
749
|
+
const conflictList = conflictCols.map((c) => `"${c}"`).join(", ");
|
|
750
|
+
const updateCols = softDelete ? [...colNames, "deleted_at", "synced_at"] : [...colNames, "synced_at"];
|
|
751
|
+
const updateSet = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
|
|
752
|
+
return {
|
|
753
|
+
sql: `INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateSet}`,
|
|
754
|
+
params: values
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
buildDelete(dest, deleteIds, softDelete) {
|
|
758
|
+
if (softDelete) {
|
|
759
|
+
return {
|
|
760
|
+
sql: `UPDATE "${dest}" SET deleted_at = now(), synced_at = now() WHERE row_id = ANY($1)`,
|
|
761
|
+
params: [deleteIds]
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
return {
|
|
765
|
+
sql: `DELETE FROM "${dest}" WHERE row_id = ANY($1)`,
|
|
766
|
+
params: [deleteIds]
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
};
|
|
617
770
|
var PostgresAdapter = class {
|
|
618
771
|
/** @internal */
|
|
619
772
|
pool;
|
|
773
|
+
dialect = new PostgresSqlDialect();
|
|
620
774
|
constructor(config) {
|
|
621
775
|
const poolConfig = {
|
|
622
776
|
connectionString: config.connectionString
|
|
623
777
|
};
|
|
624
778
|
this.pool = new Pool(poolConfig);
|
|
625
779
|
}
|
|
780
|
+
get executor() {
|
|
781
|
+
const pool = this.pool;
|
|
782
|
+
return {
|
|
783
|
+
async query(sql, params) {
|
|
784
|
+
await pool.query(sql, params);
|
|
785
|
+
},
|
|
786
|
+
async queryRows(sql, params) {
|
|
787
|
+
const result = await pool.query(sql, params);
|
|
788
|
+
return result.rows;
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
}
|
|
626
792
|
/**
|
|
627
793
|
* Insert deltas into the database in a single batch.
|
|
628
794
|
* Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
|
|
@@ -719,85 +885,11 @@ CREATE INDEX IF NOT EXISTS idx_lakesync_deltas_table_row ON lakesync_deltas ("ta
|
|
|
719
885
|
/**
|
|
720
886
|
* Materialise deltas into destination tables.
|
|
721
887
|
*
|
|
722
|
-
*
|
|
723
|
-
*
|
|
724
|
-
* rows are deleted. The `props` column is never touched.
|
|
888
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
889
|
+
* Postgres SQL dialect.
|
|
725
890
|
*/
|
|
726
891
|
async materialise(deltas, schemas) {
|
|
727
|
-
|
|
728
|
-
return Ok(void 0);
|
|
729
|
-
}
|
|
730
|
-
return wrapAsync(async () => {
|
|
731
|
-
const grouped = groupDeltasByTable(deltas);
|
|
732
|
-
const schemaIndex = buildSchemaIndex(schemas);
|
|
733
|
-
for (const [tableName, rowIds] of grouped) {
|
|
734
|
-
const schema = schemaIndex.get(tableName);
|
|
735
|
-
if (!schema) continue;
|
|
736
|
-
const dest = schema.table;
|
|
737
|
-
const columnDefs = schema.columns.map((c) => `"${c.name}" ${POSTGRES_TYPE_MAP[c.type]}`).join(", ");
|
|
738
|
-
await this.pool.query(
|
|
739
|
-
`CREATE TABLE IF NOT EXISTS "${dest}" (
|
|
740
|
-
row_id TEXT PRIMARY KEY,
|
|
741
|
-
${columnDefs},
|
|
742
|
-
props JSONB NOT NULL DEFAULT '{}',
|
|
743
|
-
synced_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
744
|
-
)`
|
|
745
|
-
);
|
|
746
|
-
const sourceTable = schema.sourceTable ?? schema.table;
|
|
747
|
-
const rowIdArray = [...rowIds];
|
|
748
|
-
const deltaResult = await this.pool.query(
|
|
749
|
-
`SELECT row_id, columns, op FROM lakesync_deltas WHERE "table" = $1 AND row_id = ANY($2) ORDER BY hlc ASC`,
|
|
750
|
-
[sourceTable, rowIdArray]
|
|
751
|
-
);
|
|
752
|
-
const byRowId = /* @__PURE__ */ new Map();
|
|
753
|
-
for (const row of deltaResult.rows) {
|
|
754
|
-
const rid = row.row_id;
|
|
755
|
-
let arr = byRowId.get(rid);
|
|
756
|
-
if (!arr) {
|
|
757
|
-
arr = [];
|
|
758
|
-
byRowId.set(rid, arr);
|
|
759
|
-
}
|
|
760
|
-
arr.push(row);
|
|
761
|
-
}
|
|
762
|
-
const upserts = [];
|
|
763
|
-
const deleteIds = [];
|
|
764
|
-
for (const [rowId, rows] of byRowId) {
|
|
765
|
-
const state = mergeLatestState(rows);
|
|
766
|
-
if (state !== null) {
|
|
767
|
-
upserts.push({ rowId, state });
|
|
768
|
-
} else {
|
|
769
|
-
deleteIds.push(rowId);
|
|
770
|
-
}
|
|
771
|
-
}
|
|
772
|
-
if (upserts.length > 0) {
|
|
773
|
-
const colNames = schema.columns.map((c) => c.name);
|
|
774
|
-
const allCols = ["row_id", ...colNames, "synced_at"];
|
|
775
|
-
const colList = allCols.map((c) => `"${c}"`).join(", ");
|
|
776
|
-
const values = [];
|
|
777
|
-
const valueRows = [];
|
|
778
|
-
const paramsPerRow = allCols.length;
|
|
779
|
-
for (let i = 0; i < upserts.length; i++) {
|
|
780
|
-
const u = upserts[i];
|
|
781
|
-
const offset = i * paramsPerRow;
|
|
782
|
-
const placeholders = allCols.map((_, j) => `$${offset + j + 1}`);
|
|
783
|
-
valueRows.push(`(${placeholders.join(", ")})`);
|
|
784
|
-
values.push(u.rowId);
|
|
785
|
-
for (const col of colNames) {
|
|
786
|
-
values.push(u.state[col] ?? null);
|
|
787
|
-
}
|
|
788
|
-
values.push(/* @__PURE__ */ new Date());
|
|
789
|
-
}
|
|
790
|
-
const updateSet = [...colNames, "synced_at"].map((c) => `"${c}" = EXCLUDED."${c}"`).join(", ");
|
|
791
|
-
await this.pool.query(
|
|
792
|
-
`INSERT INTO "${dest}" (${colList}) VALUES ${valueRows.join(", ")} ON CONFLICT (row_id) DO UPDATE SET ${updateSet}`,
|
|
793
|
-
values
|
|
794
|
-
);
|
|
795
|
-
}
|
|
796
|
-
if (deleteIds.length > 0) {
|
|
797
|
-
await this.pool.query(`DELETE FROM "${dest}" WHERE row_id = ANY($1)`, [deleteIds]);
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
}, "Failed to materialise deltas");
|
|
892
|
+
return executeMaterialise(this.executor, this.dialect, deltas, schemas);
|
|
801
893
|
}
|
|
802
894
|
/** Close the database connection pool and release resources. */
|
|
803
895
|
async close() {
|
|
@@ -821,30 +913,19 @@ function rowToRowDelta2(row) {
|
|
|
821
913
|
function createDatabaseAdapter(config) {
|
|
822
914
|
try {
|
|
823
915
|
switch (config.type) {
|
|
824
|
-
case "postgres":
|
|
825
|
-
if (!config.postgres) {
|
|
826
|
-
return Err(new AdapterError("Postgres connector config missing postgres field"));
|
|
827
|
-
}
|
|
916
|
+
case "postgres":
|
|
828
917
|
return Ok(
|
|
829
918
|
new PostgresAdapter({
|
|
830
919
|
connectionString: config.postgres.connectionString
|
|
831
920
|
})
|
|
832
921
|
);
|
|
833
|
-
|
|
834
|
-
case "mysql": {
|
|
835
|
-
if (!config.mysql) {
|
|
836
|
-
return Err(new AdapterError("MySQL connector config missing mysql field"));
|
|
837
|
-
}
|
|
922
|
+
case "mysql":
|
|
838
923
|
return Ok(
|
|
839
924
|
new MySQLAdapter({
|
|
840
925
|
connectionString: config.mysql.connectionString
|
|
841
926
|
})
|
|
842
927
|
);
|
|
843
|
-
|
|
844
|
-
case "bigquery": {
|
|
845
|
-
if (!config.bigquery) {
|
|
846
|
-
return Err(new AdapterError("BigQuery connector config missing bigquery field"));
|
|
847
|
-
}
|
|
928
|
+
case "bigquery":
|
|
848
929
|
return Ok(
|
|
849
930
|
new BigQueryAdapter({
|
|
850
931
|
projectId: config.bigquery.projectId,
|
|
@@ -853,9 +934,11 @@ function createDatabaseAdapter(config) {
|
|
|
853
934
|
location: config.bigquery.location
|
|
854
935
|
})
|
|
855
936
|
);
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
return Err(
|
|
937
|
+
case "jira":
|
|
938
|
+
case "salesforce":
|
|
939
|
+
return Err(
|
|
940
|
+
new AdapterError(`Connector type "${config.type}" does not use a DatabaseAdapter`)
|
|
941
|
+
);
|
|
859
942
|
}
|
|
860
943
|
} catch (err) {
|
|
861
944
|
return Err(new AdapterError(`Failed to create adapter: ${toError(err).message}`));
|
|
@@ -1151,7 +1234,6 @@ var MinIOAdapter = class {
|
|
|
1151
1234
|
async function createQueryFn(config) {
|
|
1152
1235
|
switch (config.type) {
|
|
1153
1236
|
case "postgres": {
|
|
1154
|
-
if (!config.postgres) return null;
|
|
1155
1237
|
const { Pool: Pool2 } = await import("pg");
|
|
1156
1238
|
const pool = new Pool2({ connectionString: config.postgres.connectionString });
|
|
1157
1239
|
return async (sql, params) => {
|
|
@@ -1160,7 +1242,6 @@ async function createQueryFn(config) {
|
|
|
1160
1242
|
};
|
|
1161
1243
|
}
|
|
1162
1244
|
case "mysql": {
|
|
1163
|
-
if (!config.mysql) return null;
|
|
1164
1245
|
const mysql2 = await import("mysql2/promise");
|
|
1165
1246
|
const pool = mysql2.createPool(config.mysql.connectionString);
|
|
1166
1247
|
return async (sql, params) => {
|
|
@@ -1176,15 +1257,23 @@ async function createQueryFn(config) {
|
|
|
1176
1257
|
export {
|
|
1177
1258
|
lakeSyncTypeToBigQuery,
|
|
1178
1259
|
isDatabaseAdapter,
|
|
1179
|
-
|
|
1180
|
-
groupDeltasByTable,
|
|
1181
|
-
buildSchemaIndex,
|
|
1260
|
+
groupAndMerge,
|
|
1182
1261
|
toCause,
|
|
1183
1262
|
wrapAsync,
|
|
1184
1263
|
mergeLatestState,
|
|
1264
|
+
isMaterialisable,
|
|
1265
|
+
resolvePrimaryKey,
|
|
1266
|
+
resolveConflictColumns,
|
|
1267
|
+
isSoftDelete,
|
|
1268
|
+
groupDeltasByTable,
|
|
1269
|
+
buildSchemaIndex,
|
|
1270
|
+
executeMaterialise,
|
|
1271
|
+
BigQuerySqlDialect,
|
|
1185
1272
|
BigQueryAdapter,
|
|
1186
1273
|
CompositeAdapter,
|
|
1274
|
+
MySqlDialect,
|
|
1187
1275
|
MySQLAdapter,
|
|
1276
|
+
PostgresSqlDialect,
|
|
1188
1277
|
PostgresAdapter,
|
|
1189
1278
|
createDatabaseAdapter,
|
|
1190
1279
|
FanOutAdapter,
|
|
@@ -1194,4 +1283,4 @@ export {
|
|
|
1194
1283
|
MinIOAdapter,
|
|
1195
1284
|
createQueryFn
|
|
1196
1285
|
};
|
|
1197
|
-
//# sourceMappingURL=chunk-
|
|
1286
|
+
//# sourceMappingURL=chunk-LPWXOYNS.js.map
|