lakesync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/adapter.d.ts +369 -0
- package/dist/adapter.js +39 -0
- package/dist/adapter.js.map +1 -0
- package/dist/analyst.d.ts +268 -0
- package/dist/analyst.js +495 -0
- package/dist/analyst.js.map +1 -0
- package/dist/auth-CAVutXzx.d.ts +30 -0
- package/dist/base-poller-Qo_SmCZs.d.ts +82 -0
- package/dist/catalogue.d.ts +65 -0
- package/dist/catalogue.js +17 -0
- package/dist/catalogue.js.map +1 -0
- package/dist/chunk-4ARO6KTJ.js +257 -0
- package/dist/chunk-4ARO6KTJ.js.map +1 -0
- package/dist/chunk-5YOFCJQ7.js +1115 -0
- package/dist/chunk-5YOFCJQ7.js.map +1 -0
- package/dist/chunk-7D4SUZUM.js +38 -0
- package/dist/chunk-7D4SUZUM.js.map +1 -0
- package/dist/chunk-BNJOGBYK.js +335 -0
- package/dist/chunk-BNJOGBYK.js.map +1 -0
- package/dist/chunk-ICNT7I3K.js +1180 -0
- package/dist/chunk-ICNT7I3K.js.map +1 -0
- package/dist/chunk-P5DRFKIT.js +413 -0
- package/dist/chunk-P5DRFKIT.js.map +1 -0
- package/dist/chunk-X3RO5SYJ.js +880 -0
- package/dist/chunk-X3RO5SYJ.js.map +1 -0
- package/dist/client.d.ts +428 -0
- package/dist/client.js +2048 -0
- package/dist/client.js.map +1 -0
- package/dist/compactor.d.ts +342 -0
- package/dist/compactor.js +793 -0
- package/dist/compactor.js.map +1 -0
- package/dist/coordinator-CxckTzYW.d.ts +396 -0
- package/dist/db-types-BR6Kt4uf.d.ts +29 -0
- package/dist/gateway-D5SaaMvT.d.ts +337 -0
- package/dist/gateway-server.d.ts +306 -0
- package/dist/gateway-server.js +4663 -0
- package/dist/gateway-server.js.map +1 -0
- package/dist/gateway.d.ts +196 -0
- package/dist/gateway.js +79 -0
- package/dist/gateway.js.map +1 -0
- package/dist/hlc-DiD8QNG3.d.ts +70 -0
- package/dist/index.d.ts +245 -0
- package/dist/index.js +102 -0
- package/dist/index.js.map +1 -0
- package/dist/json-dYtqiL0F.d.ts +18 -0
- package/dist/nessie-client-DrNikVXy.d.ts +160 -0
- package/dist/parquet.d.ts +78 -0
- package/dist/parquet.js +15 -0
- package/dist/parquet.js.map +1 -0
- package/dist/proto.d.ts +434 -0
- package/dist/proto.js +67 -0
- package/dist/proto.js.map +1 -0
- package/dist/react.d.ts +147 -0
- package/dist/react.js +224 -0
- package/dist/react.js.map +1 -0
- package/dist/resolver-C3Wphi6O.d.ts +10 -0
- package/dist/result-CojzlFE2.d.ts +64 -0
- package/dist/src-QU2YLPZY.js +383 -0
- package/dist/src-QU2YLPZY.js.map +1 -0
- package/dist/src-WYBF5LOI.js +102 -0
- package/dist/src-WYBF5LOI.js.map +1 -0
- package/dist/src-WZNPHANQ.js +426 -0
- package/dist/src-WZNPHANQ.js.map +1 -0
- package/dist/types-Bs-QyOe-.d.ts +143 -0
- package/dist/types-DAQL_vU_.d.ts +118 -0
- package/dist/types-DSC_EiwR.d.ts +45 -0
- package/dist/types-V_jVu2sA.d.ts +73 -0
- package/package.json +119 -0
|
@@ -0,0 +1,1180 @@
|
|
|
1
|
+
// ../core/src/result/errors.ts
|
|
2
|
+
var LakeSyncError = class extends Error {
|
|
3
|
+
code;
|
|
4
|
+
cause;
|
|
5
|
+
constructor(message, code, cause) {
|
|
6
|
+
super(message);
|
|
7
|
+
this.name = this.constructor.name;
|
|
8
|
+
this.code = code;
|
|
9
|
+
this.cause = cause;
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
var ClockDriftError = class extends LakeSyncError {
|
|
13
|
+
constructor(message, cause) {
|
|
14
|
+
super(message, "CLOCK_DRIFT", cause);
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
var ConflictError = class extends LakeSyncError {
|
|
18
|
+
constructor(message, cause) {
|
|
19
|
+
super(message, "CONFLICT", cause);
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
var FlushError = class extends LakeSyncError {
|
|
23
|
+
constructor(message, cause) {
|
|
24
|
+
super(message, "FLUSH_FAILED", cause);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
var SchemaError = class extends LakeSyncError {
|
|
28
|
+
constructor(message, cause) {
|
|
29
|
+
super(message, "SCHEMA_MISMATCH", cause);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
var AdapterError = class extends LakeSyncError {
|
|
33
|
+
constructor(message, cause) {
|
|
34
|
+
super(message, "ADAPTER_ERROR", cause);
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
var AdapterNotFoundError = class extends LakeSyncError {
|
|
38
|
+
constructor(message, cause) {
|
|
39
|
+
super(message, "ADAPTER_NOT_FOUND", cause);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
var BackpressureError = class extends LakeSyncError {
|
|
43
|
+
constructor(message, cause) {
|
|
44
|
+
super(message, "BACKPRESSURE", cause);
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
function toError(err) {
|
|
48
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// ../core/src/action/errors.ts
|
|
52
|
+
var ActionExecutionError = class extends LakeSyncError {
|
|
53
|
+
retryable;
|
|
54
|
+
constructor(message, retryable, cause) {
|
|
55
|
+
super(message, "ACTION_EXECUTION_ERROR", cause);
|
|
56
|
+
this.retryable = retryable;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
var ActionNotSupportedError = class extends LakeSyncError {
|
|
60
|
+
constructor(message, cause) {
|
|
61
|
+
super(message, "ACTION_NOT_SUPPORTED", cause);
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
var ActionValidationError = class extends LakeSyncError {
|
|
65
|
+
constructor(message, cause) {
|
|
66
|
+
super(message, "ACTION_VALIDATION_ERROR", cause);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
// ../core/src/action/generate-id.ts
|
|
71
|
+
import stableStringify from "fast-json-stable-stringify";
|
|
72
|
+
async function generateActionId(params) {
|
|
73
|
+
const payload = stableStringify({
|
|
74
|
+
clientId: params.clientId,
|
|
75
|
+
hlc: params.hlc.toString(),
|
|
76
|
+
connector: params.connector,
|
|
77
|
+
actionType: params.actionType,
|
|
78
|
+
params: params.params
|
|
79
|
+
});
|
|
80
|
+
const data = new TextEncoder().encode(payload);
|
|
81
|
+
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
|
82
|
+
const bytes = new Uint8Array(hashBuffer);
|
|
83
|
+
let hex = "";
|
|
84
|
+
for (const b of bytes) {
|
|
85
|
+
hex += b.toString(16).padStart(2, "0");
|
|
86
|
+
}
|
|
87
|
+
return hex;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// ../core/src/action/types.ts
|
|
91
|
+
function isActionError(result) {
|
|
92
|
+
return "code" in result && "retryable" in result;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ../core/src/result/result.ts
|
|
96
|
+
function Ok(value) {
|
|
97
|
+
return { ok: true, value };
|
|
98
|
+
}
|
|
99
|
+
function Err(error) {
|
|
100
|
+
return { ok: false, error };
|
|
101
|
+
}
|
|
102
|
+
function mapResult(result, fn) {
|
|
103
|
+
if (result.ok) {
|
|
104
|
+
return Ok(fn(result.value));
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
function flatMapResult(result, fn) {
|
|
109
|
+
if (result.ok) {
|
|
110
|
+
return fn(result.value);
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
function unwrapOrThrow(result) {
|
|
115
|
+
if (result.ok) {
|
|
116
|
+
return result.value;
|
|
117
|
+
}
|
|
118
|
+
throw result.error;
|
|
119
|
+
}
|
|
120
|
+
async function fromPromise(promise) {
|
|
121
|
+
try {
|
|
122
|
+
const value = await promise;
|
|
123
|
+
return Ok(value);
|
|
124
|
+
} catch (error) {
|
|
125
|
+
return Err(error instanceof Error ? error : new Error(String(error)));
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// ../core/src/action/validate.ts
|
|
130
|
+
function validateAction(action) {
|
|
131
|
+
if (action === null || typeof action !== "object") {
|
|
132
|
+
return Err(new ActionValidationError("Action must be a non-null object"));
|
|
133
|
+
}
|
|
134
|
+
const a = action;
|
|
135
|
+
if (typeof a.actionId !== "string" || a.actionId.length === 0) {
|
|
136
|
+
return Err(new ActionValidationError("actionId must be a non-empty string"));
|
|
137
|
+
}
|
|
138
|
+
if (typeof a.clientId !== "string" || a.clientId.length === 0) {
|
|
139
|
+
return Err(new ActionValidationError("clientId must be a non-empty string"));
|
|
140
|
+
}
|
|
141
|
+
if (typeof a.hlc !== "bigint") {
|
|
142
|
+
return Err(new ActionValidationError("hlc must be a bigint"));
|
|
143
|
+
}
|
|
144
|
+
if (typeof a.connector !== "string" || a.connector.length === 0) {
|
|
145
|
+
return Err(new ActionValidationError("connector must be a non-empty string"));
|
|
146
|
+
}
|
|
147
|
+
if (typeof a.actionType !== "string" || a.actionType.length === 0) {
|
|
148
|
+
return Err(new ActionValidationError("actionType must be a non-empty string"));
|
|
149
|
+
}
|
|
150
|
+
if (a.params === null || typeof a.params !== "object" || Array.isArray(a.params)) {
|
|
151
|
+
return Err(new ActionValidationError("params must be a non-null object"));
|
|
152
|
+
}
|
|
153
|
+
if (a.idempotencyKey !== void 0 && typeof a.idempotencyKey !== "string") {
|
|
154
|
+
return Err(new ActionValidationError("idempotencyKey must be a string if provided"));
|
|
155
|
+
}
|
|
156
|
+
return Ok(action);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ../core/src/auth.ts
|
|
160
|
+
var AuthError = class extends Error {
|
|
161
|
+
constructor(message) {
|
|
162
|
+
super(message);
|
|
163
|
+
this.name = "AuthError";
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
function base64urlDecode(input) {
|
|
167
|
+
const base64 = input.replace(/-/g, "+").replace(/_/g, "/");
|
|
168
|
+
const padded = base64.padEnd(base64.length + (4 - base64.length % 4) % 4, "=");
|
|
169
|
+
const binary = atob(padded);
|
|
170
|
+
const bytes = new Uint8Array(binary.length);
|
|
171
|
+
for (let i = 0; i < binary.length; i++) {
|
|
172
|
+
bytes[i] = binary.charCodeAt(i);
|
|
173
|
+
}
|
|
174
|
+
return bytes;
|
|
175
|
+
}
|
|
176
|
+
function parseJson(text) {
|
|
177
|
+
try {
|
|
178
|
+
return JSON.parse(text);
|
|
179
|
+
} catch {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function verifyToken(token, secret) {
|
|
184
|
+
const parts = token.split(".");
|
|
185
|
+
if (parts.length !== 3) {
|
|
186
|
+
return Err(new AuthError("Malformed JWT: expected three dot-separated segments"));
|
|
187
|
+
}
|
|
188
|
+
const [headerB64, payloadB64, signatureB64] = parts;
|
|
189
|
+
if (!headerB64 || !payloadB64 || !signatureB64) {
|
|
190
|
+
return Err(new AuthError("Malformed JWT: empty segment"));
|
|
191
|
+
}
|
|
192
|
+
let headerBytes;
|
|
193
|
+
try {
|
|
194
|
+
headerBytes = base64urlDecode(headerB64);
|
|
195
|
+
} catch {
|
|
196
|
+
return Err(new AuthError("Malformed JWT: invalid base64url in header"));
|
|
197
|
+
}
|
|
198
|
+
const header = parseJson(new TextDecoder().decode(headerBytes));
|
|
199
|
+
if (!header || header.alg !== "HS256" || header.typ !== "JWT") {
|
|
200
|
+
return Err(new AuthError('Unsupported JWT: header must be {"alg":"HS256","typ":"JWT"}'));
|
|
201
|
+
}
|
|
202
|
+
const encoder = new TextEncoder();
|
|
203
|
+
const keyData = encoder.encode(secret);
|
|
204
|
+
let cryptoKey;
|
|
205
|
+
try {
|
|
206
|
+
cryptoKey = await crypto.subtle.importKey(
|
|
207
|
+
"raw",
|
|
208
|
+
keyData,
|
|
209
|
+
{ name: "HMAC", hash: "SHA-256" },
|
|
210
|
+
false,
|
|
211
|
+
["verify"]
|
|
212
|
+
);
|
|
213
|
+
} catch {
|
|
214
|
+
return Err(new AuthError("Failed to import HMAC key"));
|
|
215
|
+
}
|
|
216
|
+
let signatureBytes;
|
|
217
|
+
try {
|
|
218
|
+
signatureBytes = base64urlDecode(signatureB64);
|
|
219
|
+
} catch {
|
|
220
|
+
return Err(new AuthError("Malformed JWT: invalid base64url in signature"));
|
|
221
|
+
}
|
|
222
|
+
const signingInput = encoder.encode(`${headerB64}.${payloadB64}`);
|
|
223
|
+
let valid;
|
|
224
|
+
try {
|
|
225
|
+
valid = await crypto.subtle.verify(
|
|
226
|
+
"HMAC",
|
|
227
|
+
cryptoKey,
|
|
228
|
+
signatureBytes,
|
|
229
|
+
signingInput
|
|
230
|
+
);
|
|
231
|
+
} catch {
|
|
232
|
+
return Err(new AuthError("Signature verification failed"));
|
|
233
|
+
}
|
|
234
|
+
if (!valid) {
|
|
235
|
+
return Err(new AuthError("Invalid JWT signature"));
|
|
236
|
+
}
|
|
237
|
+
let payloadBytes;
|
|
238
|
+
try {
|
|
239
|
+
payloadBytes = base64urlDecode(payloadB64);
|
|
240
|
+
} catch {
|
|
241
|
+
return Err(new AuthError("Malformed JWT: invalid base64url in payload"));
|
|
242
|
+
}
|
|
243
|
+
const payload = parseJson(new TextDecoder().decode(payloadBytes));
|
|
244
|
+
if (!payload) {
|
|
245
|
+
return Err(new AuthError("Malformed JWT: payload is not valid JSON"));
|
|
246
|
+
}
|
|
247
|
+
if (payload.exp === void 0 || typeof payload.exp !== "number") {
|
|
248
|
+
return Err(new AuthError('Missing or invalid "exp" claim (expiry)'));
|
|
249
|
+
}
|
|
250
|
+
const nowSeconds = Math.floor(Date.now() / 1e3);
|
|
251
|
+
if (payload.exp <= nowSeconds) {
|
|
252
|
+
return Err(new AuthError("JWT has expired"));
|
|
253
|
+
}
|
|
254
|
+
if (typeof payload.sub !== "string" || payload.sub.length === 0) {
|
|
255
|
+
return Err(new AuthError('Missing or invalid "sub" claim (clientId)'));
|
|
256
|
+
}
|
|
257
|
+
if (typeof payload.gw !== "string" || payload.gw.length === 0) {
|
|
258
|
+
return Err(new AuthError('Missing or invalid "gw" claim (gatewayId)'));
|
|
259
|
+
}
|
|
260
|
+
const standardClaims = /* @__PURE__ */ new Set(["sub", "gw", "exp", "iat", "iss", "aud", "role"]);
|
|
261
|
+
const customClaims = {};
|
|
262
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
263
|
+
if (standardClaims.has(key)) continue;
|
|
264
|
+
if (typeof value === "string") {
|
|
265
|
+
customClaims[key] = value;
|
|
266
|
+
} else if (Array.isArray(value) && value.every((v) => typeof v === "string")) {
|
|
267
|
+
customClaims[key] = value;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
customClaims.sub = payload.sub;
|
|
271
|
+
const role = typeof payload.role === "string" && payload.role.length > 0 ? payload.role : "client";
|
|
272
|
+
return Ok({
|
|
273
|
+
clientId: payload.sub,
|
|
274
|
+
gatewayId: payload.gw,
|
|
275
|
+
role,
|
|
276
|
+
customClaims
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// ../core/src/hlc/hlc.ts
|
|
281
|
+
var HLC = class _HLC {
|
|
282
|
+
wallClock;
|
|
283
|
+
counter = 0;
|
|
284
|
+
lastWall = 0;
|
|
285
|
+
/** Maximum tolerated drift between local and remote physical clocks (ms). */
|
|
286
|
+
static MAX_DRIFT_MS = 5e3;
|
|
287
|
+
/** Maximum value of the 16-bit logical counter. */
|
|
288
|
+
static MAX_COUNTER = 65535;
|
|
289
|
+
/**
|
|
290
|
+
* Create a new HLC instance.
|
|
291
|
+
*
|
|
292
|
+
* @param wallClock - Optional injectable clock source returning epoch ms.
|
|
293
|
+
* Defaults to `Date.now`.
|
|
294
|
+
*/
|
|
295
|
+
constructor(wallClock) {
|
|
296
|
+
this.wallClock = wallClock ?? (() => Date.now());
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Generate a new monotonically increasing HLC timestamp.
|
|
300
|
+
*
|
|
301
|
+
* The returned timestamp is guaranteed to be strictly greater than any
|
|
302
|
+
* previously returned by this instance.
|
|
303
|
+
*/
|
|
304
|
+
now() {
|
|
305
|
+
const physical = this.wallClock();
|
|
306
|
+
const wall = Math.max(physical, this.lastWall);
|
|
307
|
+
if (wall === this.lastWall) {
|
|
308
|
+
this.counter++;
|
|
309
|
+
if (this.counter > _HLC.MAX_COUNTER) {
|
|
310
|
+
this.lastWall = wall + 1;
|
|
311
|
+
this.counter = 0;
|
|
312
|
+
}
|
|
313
|
+
} else {
|
|
314
|
+
this.lastWall = wall;
|
|
315
|
+
this.counter = 0;
|
|
316
|
+
}
|
|
317
|
+
return _HLC.encode(this.lastWall, this.counter);
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Receive a remote HLC timestamp and advance the local clock.
|
|
321
|
+
*
|
|
322
|
+
* Returns `Err(ClockDriftError)` if the remote timestamp indicates
|
|
323
|
+
* clock drift exceeding {@link MAX_DRIFT_MS}.
|
|
324
|
+
*
|
|
325
|
+
* @param remote - The HLC timestamp received from a remote node.
|
|
326
|
+
* @returns A `Result` containing the new local HLC timestamp, or a
|
|
327
|
+
* `ClockDriftError` if the remote clock is too far ahead.
|
|
328
|
+
*/
|
|
329
|
+
recv(remote) {
|
|
330
|
+
const { wall: remoteWall, counter: remoteCounter } = _HLC.decode(remote);
|
|
331
|
+
const physical = this.wallClock();
|
|
332
|
+
const localWall = Math.max(physical, this.lastWall);
|
|
333
|
+
if (remoteWall - physical > _HLC.MAX_DRIFT_MS) {
|
|
334
|
+
return Err(
|
|
335
|
+
new ClockDriftError(
|
|
336
|
+
`Remote clock is ${remoteWall - physical}ms ahead (max drift: ${_HLC.MAX_DRIFT_MS}ms)`
|
|
337
|
+
)
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
if (remoteWall > localWall) {
|
|
341
|
+
this.lastWall = remoteWall;
|
|
342
|
+
this.counter = remoteCounter + 1;
|
|
343
|
+
} else if (remoteWall === localWall) {
|
|
344
|
+
this.lastWall = localWall;
|
|
345
|
+
this.counter = Math.max(this.counter, remoteCounter) + 1;
|
|
346
|
+
} else {
|
|
347
|
+
this.lastWall = localWall;
|
|
348
|
+
this.counter++;
|
|
349
|
+
}
|
|
350
|
+
if (this.counter > _HLC.MAX_COUNTER) {
|
|
351
|
+
this.lastWall = this.lastWall + 1;
|
|
352
|
+
this.counter = 0;
|
|
353
|
+
}
|
|
354
|
+
return Ok(_HLC.encode(this.lastWall, this.counter));
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Encode a wall clock value (ms) and logical counter into a 64-bit HLC timestamp.
|
|
358
|
+
*
|
|
359
|
+
* @param wall - Wall clock component in epoch milliseconds (48-bit).
|
|
360
|
+
* @param counter - Logical counter component (16-bit, 0..65535).
|
|
361
|
+
* @returns The encoded {@link HLCTimestamp}.
|
|
362
|
+
*/
|
|
363
|
+
static encode(wall, counter) {
|
|
364
|
+
return BigInt(wall) << 16n | BigInt(counter & 65535);
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Decode an HLC timestamp into its wall clock (ms) and logical counter components.
|
|
368
|
+
*
|
|
369
|
+
* @param ts - The {@link HLCTimestamp} to decode.
|
|
370
|
+
* @returns An object with `wall` (epoch ms) and `counter` (logical) fields.
|
|
371
|
+
*/
|
|
372
|
+
static decode(ts) {
|
|
373
|
+
return {
|
|
374
|
+
wall: Number(ts >> 16n),
|
|
375
|
+
counter: Number(ts & 0xffffn)
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Compare two HLC timestamps.
|
|
380
|
+
*
|
|
381
|
+
* @returns `-1` if `a < b`, `0` if `a === b`, `1` if `a > b`.
|
|
382
|
+
*/
|
|
383
|
+
static compare(a, b) {
|
|
384
|
+
if (a < b) return -1;
|
|
385
|
+
if (a > b) return 1;
|
|
386
|
+
return 0;
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
// ../core/src/base-poller.ts
|
|
391
|
+
function isIngestTarget(target) {
|
|
392
|
+
return typeof target.flush === "function" && typeof target.shouldFlush === "function" && "bufferStats" in target;
|
|
393
|
+
}
|
|
394
|
+
var DEFAULT_CHUNK_SIZE = 500;
|
|
395
|
+
var DEFAULT_FLUSH_THRESHOLD = 0.7;
|
|
396
|
+
var BaseSourcePoller = class {
|
|
397
|
+
gateway;
|
|
398
|
+
hlc;
|
|
399
|
+
clientId;
|
|
400
|
+
intervalMs;
|
|
401
|
+
timer = null;
|
|
402
|
+
running = false;
|
|
403
|
+
chunkSize;
|
|
404
|
+
memoryBudgetBytes;
|
|
405
|
+
flushThreshold;
|
|
406
|
+
pendingDeltas = [];
|
|
407
|
+
constructor(config) {
|
|
408
|
+
this.gateway = config.gateway;
|
|
409
|
+
this.hlc = new HLC();
|
|
410
|
+
this.clientId = `ingest:${config.name}`;
|
|
411
|
+
this.intervalMs = config.intervalMs;
|
|
412
|
+
this.chunkSize = config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE;
|
|
413
|
+
this.memoryBudgetBytes = config.memory?.memoryBudgetBytes;
|
|
414
|
+
this.flushThreshold = config.memory?.flushThreshold ?? DEFAULT_FLUSH_THRESHOLD;
|
|
415
|
+
}
|
|
416
|
+
/** Start the polling loop. */
|
|
417
|
+
start() {
|
|
418
|
+
if (this.running) return;
|
|
419
|
+
this.running = true;
|
|
420
|
+
this.schedulePoll();
|
|
421
|
+
}
|
|
422
|
+
/** Stop the polling loop. */
|
|
423
|
+
stop() {
|
|
424
|
+
this.running = false;
|
|
425
|
+
if (this.timer) {
|
|
426
|
+
clearTimeout(this.timer);
|
|
427
|
+
this.timer = null;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
/** Whether the poller is currently running. */
|
|
431
|
+
get isRunning() {
|
|
432
|
+
return this.running;
|
|
433
|
+
}
|
|
434
|
+
/** Push collected deltas to the gateway (single-shot, backward compat). */
|
|
435
|
+
pushDeltas(deltas) {
|
|
436
|
+
if (deltas.length === 0) return;
|
|
437
|
+
const push = {
|
|
438
|
+
clientId: this.clientId,
|
|
439
|
+
deltas,
|
|
440
|
+
lastSeenHlc: 0n
|
|
441
|
+
};
|
|
442
|
+
this.gateway.handlePush(push);
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Accumulate a single delta. When `chunkSize` is reached, the pending
|
|
446
|
+
* deltas are automatically pushed (and flushed if needed).
|
|
447
|
+
*/
|
|
448
|
+
async accumulateDelta(delta) {
|
|
449
|
+
this.pendingDeltas.push(delta);
|
|
450
|
+
if (this.pendingDeltas.length >= this.chunkSize) {
|
|
451
|
+
await this.pushPendingChunk();
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
/** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
|
|
455
|
+
async flushAccumulator() {
|
|
456
|
+
if (this.pendingDeltas.length > 0) {
|
|
457
|
+
await this.pushPendingChunk();
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Push a chunk of pending deltas. If the gateway is an IngestTarget,
|
|
462
|
+
* checks memory pressure and flushes before/after push when needed.
|
|
463
|
+
* On backpressure, flushes once and retries.
|
|
464
|
+
*/
|
|
465
|
+
async pushPendingChunk() {
|
|
466
|
+
const chunk = this.pendingDeltas;
|
|
467
|
+
this.pendingDeltas = [];
|
|
468
|
+
await this.pushChunkWithFlush(chunk);
|
|
469
|
+
}
|
|
470
|
+
async pushChunkWithFlush(chunk) {
|
|
471
|
+
if (chunk.length === 0) return;
|
|
472
|
+
const target = this.gateway;
|
|
473
|
+
if (isIngestTarget(target)) {
|
|
474
|
+
if (this.shouldFlushTarget(target)) {
|
|
475
|
+
await target.flush();
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
const push = {
|
|
479
|
+
clientId: this.clientId,
|
|
480
|
+
deltas: chunk,
|
|
481
|
+
lastSeenHlc: 0n
|
|
482
|
+
};
|
|
483
|
+
const result = target.handlePush(push);
|
|
484
|
+
if (result && typeof result === "object" && "ok" in result && !result.ok) {
|
|
485
|
+
if (isIngestTarget(target)) {
|
|
486
|
+
await target.flush();
|
|
487
|
+
target.handlePush(push);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
shouldFlushTarget(target) {
|
|
492
|
+
if (target.shouldFlush()) return true;
|
|
493
|
+
if (this.memoryBudgetBytes != null) {
|
|
494
|
+
const threshold = Math.floor(this.memoryBudgetBytes * this.flushThreshold);
|
|
495
|
+
if (target.bufferStats.byteSize >= threshold) return true;
|
|
496
|
+
}
|
|
497
|
+
return false;
|
|
498
|
+
}
|
|
499
|
+
schedulePoll() {
|
|
500
|
+
if (!this.running) return;
|
|
501
|
+
this.timer = setTimeout(async () => {
|
|
502
|
+
try {
|
|
503
|
+
await this.poll();
|
|
504
|
+
} catch {
|
|
505
|
+
}
|
|
506
|
+
this.schedulePoll();
|
|
507
|
+
}, this.intervalMs);
|
|
508
|
+
}
|
|
509
|
+
};
|
|
510
|
+
|
|
511
|
+
// ../core/src/conflict/lww.ts
|
|
512
|
+
var LWWResolver = class {
|
|
513
|
+
/**
|
|
514
|
+
* Resolve two conflicting deltas for the same row, returning the merged result.
|
|
515
|
+
*
|
|
516
|
+
* Rules:
|
|
517
|
+
* - Both DELETE: the delta with the higher HLC (or clientId tiebreak) wins.
|
|
518
|
+
* - One DELETE, one non-DELETE: the delta with the higher HLC wins.
|
|
519
|
+
* If the DELETE wins, the row is tombstoned (empty columns).
|
|
520
|
+
* If the non-DELETE wins, the row is resurrected.
|
|
521
|
+
* - Both non-DELETE: columns are merged per-column using LWW semantics.
|
|
522
|
+
*
|
|
523
|
+
* @param local - The locally held delta for this row.
|
|
524
|
+
* @param remote - The incoming remote delta for this row.
|
|
525
|
+
* @returns A `Result` containing the resolved `RowDelta`, or a
|
|
526
|
+
* `ConflictError` if the deltas refer to different tables/rows.
|
|
527
|
+
*/
|
|
528
|
+
resolve(local, remote) {
|
|
529
|
+
if (local.table !== remote.table || local.rowId !== remote.rowId) {
|
|
530
|
+
return Err(
|
|
531
|
+
new ConflictError(
|
|
532
|
+
`Cannot resolve conflict: mismatched table/rowId (${local.table}:${local.rowId} vs ${remote.table}:${remote.rowId})`
|
|
533
|
+
)
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
const winner = pickWinner(local, remote);
|
|
537
|
+
if (local.op === "DELETE" && remote.op === "DELETE") {
|
|
538
|
+
return Ok({ ...winner, columns: [] });
|
|
539
|
+
}
|
|
540
|
+
if (local.op === "DELETE" || remote.op === "DELETE") {
|
|
541
|
+
const deleteDelta = local.op === "DELETE" ? local : remote;
|
|
542
|
+
const otherDelta = local.op === "DELETE" ? remote : local;
|
|
543
|
+
if (deleteDelta === winner) {
|
|
544
|
+
return Ok({ ...deleteDelta, columns: [] });
|
|
545
|
+
}
|
|
546
|
+
return Ok({ ...otherDelta });
|
|
547
|
+
}
|
|
548
|
+
const mergedColumns = mergeColumns(local, remote);
|
|
549
|
+
const op = local.op === "INSERT" && remote.op === "INSERT" ? "INSERT" : "UPDATE";
|
|
550
|
+
return Ok({
|
|
551
|
+
op,
|
|
552
|
+
table: local.table,
|
|
553
|
+
rowId: local.rowId,
|
|
554
|
+
clientId: winner.clientId,
|
|
555
|
+
columns: mergedColumns,
|
|
556
|
+
hlc: winner.hlc,
|
|
557
|
+
deltaId: winner.deltaId
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
function pickWinner(local, remote) {
|
|
562
|
+
const cmp = HLC.compare(local.hlc, remote.hlc);
|
|
563
|
+
if (cmp > 0) return local;
|
|
564
|
+
if (cmp < 0) return remote;
|
|
565
|
+
return local.clientId > remote.clientId ? local : remote;
|
|
566
|
+
}
|
|
567
|
+
function mergeColumns(local, remote) {
|
|
568
|
+
const localMap = new Map(local.columns.map((c) => [c.column, c]));
|
|
569
|
+
const remoteMap = new Map(remote.columns.map((c) => [c.column, c]));
|
|
570
|
+
const allColumns = /* @__PURE__ */ new Set([...localMap.keys(), ...remoteMap.keys()]);
|
|
571
|
+
const winner = pickWinner(local, remote);
|
|
572
|
+
const merged = [];
|
|
573
|
+
for (const col of allColumns) {
|
|
574
|
+
const localCol = localMap.get(col);
|
|
575
|
+
const remoteCol = remoteMap.get(col);
|
|
576
|
+
if (!remoteCol) {
|
|
577
|
+
merged.push(localCol);
|
|
578
|
+
} else if (!localCol) {
|
|
579
|
+
merged.push(remoteCol);
|
|
580
|
+
} else {
|
|
581
|
+
merged.push(winner === local ? localCol : remoteCol);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
return merged;
|
|
585
|
+
}
|
|
586
|
+
var _singleton = new LWWResolver();
|
|
587
|
+
function resolveLWW(local, remote) {
|
|
588
|
+
return _singleton.resolve(local, remote);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// ../core/src/connector/action-handler.ts
|
|
592
|
+
function isActionHandler(obj) {
|
|
593
|
+
if (obj === null || typeof obj !== "object") return false;
|
|
594
|
+
const candidate = obj;
|
|
595
|
+
return Array.isArray(candidate.supportedActions) && typeof candidate.executeAction === "function";
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// ../core/src/connector/errors.ts
|
|
599
|
+
var ConnectorValidationError = class extends LakeSyncError {
|
|
600
|
+
constructor(message, cause) {
|
|
601
|
+
super(message, "CONNECTOR_VALIDATION", cause);
|
|
602
|
+
}
|
|
603
|
+
};
|
|
604
|
+
|
|
605
|
+
// ../core/src/connector/types.ts
|
|
606
|
+
var CONNECTOR_TYPES = ["postgres", "mysql", "bigquery", "jira", "salesforce"];
|
|
607
|
+
|
|
608
|
+
// ../core/src/connector/validate.ts
|
|
609
|
+
var VALID_STRATEGIES = /* @__PURE__ */ new Set(["cursor", "diff"]);
|
|
610
|
+
function validateConnectorConfig(input) {
|
|
611
|
+
if (typeof input !== "object" || input === null) {
|
|
612
|
+
return Err(new ConnectorValidationError("Connector config must be an object"));
|
|
613
|
+
}
|
|
614
|
+
const obj = input;
|
|
615
|
+
if (typeof obj.name !== "string" || obj.name.length === 0) {
|
|
616
|
+
return Err(new ConnectorValidationError("Connector name must be a non-empty string"));
|
|
617
|
+
}
|
|
618
|
+
if (typeof obj.type !== "string" || !CONNECTOR_TYPES.includes(obj.type)) {
|
|
619
|
+
return Err(
|
|
620
|
+
new ConnectorValidationError(`Connector type must be one of: ${CONNECTOR_TYPES.join(", ")}`)
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
const connectorType = obj.type;
|
|
624
|
+
switch (connectorType) {
|
|
625
|
+
case "postgres": {
|
|
626
|
+
const pg = obj.postgres;
|
|
627
|
+
if (typeof pg !== "object" || pg === null) {
|
|
628
|
+
return Err(
|
|
629
|
+
new ConnectorValidationError(
|
|
630
|
+
'Connector type "postgres" requires a postgres config object'
|
|
631
|
+
)
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
const pgObj = pg;
|
|
635
|
+
if (typeof pgObj.connectionString !== "string" || pgObj.connectionString.length === 0) {
|
|
636
|
+
return Err(
|
|
637
|
+
new ConnectorValidationError("Postgres connector requires a non-empty connectionString")
|
|
638
|
+
);
|
|
639
|
+
}
|
|
640
|
+
break;
|
|
641
|
+
}
|
|
642
|
+
case "mysql": {
|
|
643
|
+
const my = obj.mysql;
|
|
644
|
+
if (typeof my !== "object" || my === null) {
|
|
645
|
+
return Err(
|
|
646
|
+
new ConnectorValidationError('Connector type "mysql" requires a mysql config object')
|
|
647
|
+
);
|
|
648
|
+
}
|
|
649
|
+
const myObj = my;
|
|
650
|
+
if (typeof myObj.connectionString !== "string" || myObj.connectionString.length === 0) {
|
|
651
|
+
return Err(
|
|
652
|
+
new ConnectorValidationError("MySQL connector requires a non-empty connectionString")
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
break;
|
|
656
|
+
}
|
|
657
|
+
case "bigquery": {
|
|
658
|
+
const bq = obj.bigquery;
|
|
659
|
+
if (typeof bq !== "object" || bq === null) {
|
|
660
|
+
return Err(
|
|
661
|
+
new ConnectorValidationError(
|
|
662
|
+
'Connector type "bigquery" requires a bigquery config object'
|
|
663
|
+
)
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
const bqObj = bq;
|
|
667
|
+
if (typeof bqObj.projectId !== "string" || bqObj.projectId.length === 0) {
|
|
668
|
+
return Err(
|
|
669
|
+
new ConnectorValidationError("BigQuery connector requires a non-empty projectId")
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
if (typeof bqObj.dataset !== "string" || bqObj.dataset.length === 0) {
|
|
673
|
+
return Err(new ConnectorValidationError("BigQuery connector requires a non-empty dataset"));
|
|
674
|
+
}
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
case "jira": {
|
|
678
|
+
const jira = obj.jira;
|
|
679
|
+
if (typeof jira !== "object" || jira === null) {
|
|
680
|
+
return Err(
|
|
681
|
+
new ConnectorValidationError('Connector type "jira" requires a jira config object')
|
|
682
|
+
);
|
|
683
|
+
}
|
|
684
|
+
const jiraObj = jira;
|
|
685
|
+
if (typeof jiraObj.domain !== "string" || jiraObj.domain.length === 0) {
|
|
686
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty domain"));
|
|
687
|
+
}
|
|
688
|
+
if (typeof jiraObj.email !== "string" || jiraObj.email.length === 0) {
|
|
689
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty email"));
|
|
690
|
+
}
|
|
691
|
+
if (typeof jiraObj.apiToken !== "string" || jiraObj.apiToken.length === 0) {
|
|
692
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty apiToken"));
|
|
693
|
+
}
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
case "salesforce": {
|
|
697
|
+
const sf = obj.salesforce;
|
|
698
|
+
if (typeof sf !== "object" || sf === null) {
|
|
699
|
+
return Err(
|
|
700
|
+
new ConnectorValidationError(
|
|
701
|
+
'Connector type "salesforce" requires a salesforce config object'
|
|
702
|
+
)
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
const sfObj = sf;
|
|
706
|
+
if (typeof sfObj.instanceUrl !== "string" || sfObj.instanceUrl.length === 0) {
|
|
707
|
+
return Err(
|
|
708
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty instanceUrl")
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
if (typeof sfObj.clientId !== "string" || sfObj.clientId.length === 0) {
|
|
712
|
+
return Err(
|
|
713
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty clientId")
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
if (typeof sfObj.clientSecret !== "string" || sfObj.clientSecret.length === 0) {
|
|
717
|
+
return Err(
|
|
718
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty clientSecret")
|
|
719
|
+
);
|
|
720
|
+
}
|
|
721
|
+
if (typeof sfObj.username !== "string" || sfObj.username.length === 0) {
|
|
722
|
+
return Err(
|
|
723
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty username")
|
|
724
|
+
);
|
|
725
|
+
}
|
|
726
|
+
if (typeof sfObj.password !== "string" || sfObj.password.length === 0) {
|
|
727
|
+
return Err(
|
|
728
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty password")
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
break;
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
if (obj.ingest !== void 0) {
|
|
735
|
+
if (typeof obj.ingest !== "object" || obj.ingest === null) {
|
|
736
|
+
return Err(new ConnectorValidationError("Ingest config must be an object"));
|
|
737
|
+
}
|
|
738
|
+
const ingest = obj.ingest;
|
|
739
|
+
if (connectorType === "jira" || connectorType === "salesforce") {
|
|
740
|
+
if (ingest.intervalMs !== void 0) {
|
|
741
|
+
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
742
|
+
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
return Ok(input);
|
|
746
|
+
}
|
|
747
|
+
if (!Array.isArray(ingest.tables) || ingest.tables.length === 0) {
|
|
748
|
+
return Err(new ConnectorValidationError("Ingest config must have a non-empty tables array"));
|
|
749
|
+
}
|
|
750
|
+
for (let i = 0; i < ingest.tables.length; i++) {
|
|
751
|
+
const table = ingest.tables[i];
|
|
752
|
+
if (typeof table !== "object" || table === null) {
|
|
753
|
+
return Err(new ConnectorValidationError(`Ingest table at index ${i} must be an object`));
|
|
754
|
+
}
|
|
755
|
+
if (typeof table.table !== "string" || table.table.length === 0) {
|
|
756
|
+
return Err(
|
|
757
|
+
new ConnectorValidationError(
|
|
758
|
+
`Ingest table at index ${i} must have a non-empty table name`
|
|
759
|
+
)
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
if (typeof table.query !== "string" || table.query.length === 0) {
|
|
763
|
+
return Err(
|
|
764
|
+
new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty query`)
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
if (typeof table.strategy !== "object" || table.strategy === null) {
|
|
768
|
+
return Err(
|
|
769
|
+
new ConnectorValidationError(`Ingest table at index ${i} must have a strategy object`)
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
const strategy = table.strategy;
|
|
773
|
+
if (!VALID_STRATEGIES.has(strategy.type)) {
|
|
774
|
+
return Err(
|
|
775
|
+
new ConnectorValidationError(
|
|
776
|
+
`Ingest table at index ${i} strategy type must be "cursor" or "diff"`
|
|
777
|
+
)
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
if (strategy.type === "cursor") {
|
|
781
|
+
if (typeof strategy.cursorColumn !== "string" || strategy.cursorColumn.length === 0) {
|
|
782
|
+
return Err(
|
|
783
|
+
new ConnectorValidationError(
|
|
784
|
+
`Ingest table at index ${i} cursor strategy requires a non-empty cursorColumn`
|
|
785
|
+
)
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
if (ingest.intervalMs !== void 0) {
|
|
791
|
+
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
792
|
+
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
return Ok(input);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
// ../core/src/delta/apply.ts
|
|
800
|
+
function applyDelta(row, delta) {
|
|
801
|
+
if (delta.op === "DELETE") return null;
|
|
802
|
+
const base = row ? { ...row } : {};
|
|
803
|
+
for (const col of delta.columns) {
|
|
804
|
+
base[col.column] = col.value;
|
|
805
|
+
}
|
|
806
|
+
return base;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// ../core/src/delta/extract.ts
|
|
810
|
+
import equal from "fast-deep-equal";
|
|
811
|
+
import stableStringify2 from "fast-json-stable-stringify";
|
|
812
|
+
async function extractDelta(before, after, opts) {
|
|
813
|
+
const { table, rowId, clientId, hlc, schema } = opts;
|
|
814
|
+
const beforeExists = before != null;
|
|
815
|
+
const afterExists = after != null;
|
|
816
|
+
if (!beforeExists && !afterExists) {
|
|
817
|
+
return null;
|
|
818
|
+
}
|
|
819
|
+
if (!beforeExists && afterExists) {
|
|
820
|
+
const columns2 = buildColumns(after, schema);
|
|
821
|
+
const deltaId2 = await generateDeltaId({ clientId, hlc, table, rowId, columns: columns2 });
|
|
822
|
+
return { op: "INSERT", table, rowId, clientId, columns: columns2, hlc, deltaId: deltaId2 };
|
|
823
|
+
}
|
|
824
|
+
if (beforeExists && !afterExists) {
|
|
825
|
+
const columns2 = [];
|
|
826
|
+
const deltaId2 = await generateDeltaId({ clientId, hlc, table, rowId, columns: columns2 });
|
|
827
|
+
return { op: "DELETE", table, rowId, clientId, columns: columns2, hlc, deltaId: deltaId2 };
|
|
828
|
+
}
|
|
829
|
+
const columns = diffColumns(before, after, schema);
|
|
830
|
+
if (columns.length === 0) {
|
|
831
|
+
return null;
|
|
832
|
+
}
|
|
833
|
+
const deltaId = await generateDeltaId({ clientId, hlc, table, rowId, columns });
|
|
834
|
+
return { op: "UPDATE", table, rowId, clientId, columns, hlc, deltaId };
|
|
835
|
+
}
|
|
836
|
+
function allowedSet(schema) {
|
|
837
|
+
return schema ? new Set(schema.columns.map((c) => c.name)) : null;
|
|
838
|
+
}
|
|
839
|
+
function buildColumns(row, schema) {
|
|
840
|
+
const allowed = allowedSet(schema);
|
|
841
|
+
const columns = [];
|
|
842
|
+
for (const [key, value] of Object.entries(row)) {
|
|
843
|
+
if (value === void 0) continue;
|
|
844
|
+
if (allowed && !allowed.has(key)) continue;
|
|
845
|
+
columns.push({ column: key, value });
|
|
846
|
+
}
|
|
847
|
+
return columns;
|
|
848
|
+
}
|
|
849
|
+
function diffColumns(before, after, schema) {
|
|
850
|
+
const allowed = allowedSet(schema);
|
|
851
|
+
const allKeys = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
852
|
+
const columns = [];
|
|
853
|
+
for (const key of allKeys) {
|
|
854
|
+
if (allowed && !allowed.has(key)) continue;
|
|
855
|
+
const beforeVal = before[key];
|
|
856
|
+
const afterVal = after[key];
|
|
857
|
+
if (afterVal === void 0) continue;
|
|
858
|
+
if (beforeVal === void 0) {
|
|
859
|
+
columns.push({ column: key, value: afterVal });
|
|
860
|
+
continue;
|
|
861
|
+
}
|
|
862
|
+
if (Object.is(beforeVal, afterVal)) continue;
|
|
863
|
+
if (typeof beforeVal === "object" && beforeVal !== null && typeof afterVal === "object" && afterVal !== null && equal(beforeVal, afterVal)) {
|
|
864
|
+
continue;
|
|
865
|
+
}
|
|
866
|
+
columns.push({ column: key, value: afterVal });
|
|
867
|
+
}
|
|
868
|
+
return columns;
|
|
869
|
+
}
|
|
870
|
+
async function generateDeltaId(params) {
|
|
871
|
+
const payload = stableStringify2({
|
|
872
|
+
clientId: params.clientId,
|
|
873
|
+
hlc: params.hlc.toString(),
|
|
874
|
+
table: params.table,
|
|
875
|
+
rowId: params.rowId,
|
|
876
|
+
columns: params.columns
|
|
877
|
+
});
|
|
878
|
+
const data = new TextEncoder().encode(payload);
|
|
879
|
+
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
|
880
|
+
const bytes = new Uint8Array(hashBuffer);
|
|
881
|
+
let hex = "";
|
|
882
|
+
for (const b of bytes) {
|
|
883
|
+
hex += b.toString(16).padStart(2, "0");
|
|
884
|
+
}
|
|
885
|
+
return hex;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// ../core/src/delta/types.ts
|
|
889
|
+
function rowKey(table, rowId) {
|
|
890
|
+
return `${table}:${rowId}`;
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// ../core/src/json.ts
|
|
894
|
+
function bigintReplacer(_key, value) {
|
|
895
|
+
return typeof value === "bigint" ? value.toString() : value;
|
|
896
|
+
}
|
|
897
|
+
function bigintReviver(key, value) {
|
|
898
|
+
if (typeof value === "string" && /hlc$/i.test(key)) {
|
|
899
|
+
try {
|
|
900
|
+
return BigInt(value);
|
|
901
|
+
} catch {
|
|
902
|
+
return value;
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
return value;
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
// ../core/src/sync-rules/defaults.ts
|
|
909
|
+
function createPassAllRules() {
|
|
910
|
+
return {
|
|
911
|
+
version: 1,
|
|
912
|
+
buckets: [
|
|
913
|
+
{
|
|
914
|
+
name: "all",
|
|
915
|
+
tables: [],
|
|
916
|
+
filters: []
|
|
917
|
+
}
|
|
918
|
+
]
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
function createUserScopedRules(tables, userColumn = "user_id") {
|
|
922
|
+
return {
|
|
923
|
+
version: 1,
|
|
924
|
+
buckets: [
|
|
925
|
+
{
|
|
926
|
+
name: "user",
|
|
927
|
+
tables,
|
|
928
|
+
filters: [
|
|
929
|
+
{
|
|
930
|
+
column: userColumn,
|
|
931
|
+
op: "eq",
|
|
932
|
+
value: "jwt:sub"
|
|
933
|
+
}
|
|
934
|
+
]
|
|
935
|
+
}
|
|
936
|
+
]
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
// ../core/src/sync-rules/errors.ts
|
|
941
|
+
var SyncRuleError = class extends LakeSyncError {
|
|
942
|
+
constructor(message, cause) {
|
|
943
|
+
super(message, "SYNC_RULE_ERROR", cause);
|
|
944
|
+
}
|
|
945
|
+
};
|
|
946
|
+
|
|
947
|
+
// ../core/src/sync-rules/evaluator.ts
|
|
948
|
+
function resolveFilterValue(value, claims) {
|
|
949
|
+
if (!value.startsWith("jwt:")) {
|
|
950
|
+
return [value];
|
|
951
|
+
}
|
|
952
|
+
const claimKey = value.slice(4);
|
|
953
|
+
const claimValue = claims[claimKey];
|
|
954
|
+
if (claimValue === void 0) {
|
|
955
|
+
return [];
|
|
956
|
+
}
|
|
957
|
+
return Array.isArray(claimValue) ? claimValue : [claimValue];
|
|
958
|
+
}
|
|
959
|
+
function deltaMatchesBucket(delta, bucket, claims) {
|
|
960
|
+
if (bucket.tables.length > 0 && !bucket.tables.includes(delta.table)) {
|
|
961
|
+
return false;
|
|
962
|
+
}
|
|
963
|
+
for (const filter of bucket.filters) {
|
|
964
|
+
if (!filterMatchesDelta(delta, filter, claims)) {
|
|
965
|
+
return false;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
return true;
|
|
969
|
+
}
|
|
970
|
+
function compareValues(deltaValue, filterValue, op) {
|
|
971
|
+
const numDelta = parseFloat(deltaValue);
|
|
972
|
+
const numFilter = parseFloat(filterValue);
|
|
973
|
+
const useNumeric = !Number.isNaN(numDelta) && !Number.isNaN(numFilter);
|
|
974
|
+
if (useNumeric) {
|
|
975
|
+
switch (op) {
|
|
976
|
+
case "gt":
|
|
977
|
+
return numDelta > numFilter;
|
|
978
|
+
case "lt":
|
|
979
|
+
return numDelta < numFilter;
|
|
980
|
+
case "gte":
|
|
981
|
+
return numDelta >= numFilter;
|
|
982
|
+
case "lte":
|
|
983
|
+
return numDelta <= numFilter;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
const cmp = deltaValue.localeCompare(filterValue);
|
|
987
|
+
switch (op) {
|
|
988
|
+
case "gt":
|
|
989
|
+
return cmp > 0;
|
|
990
|
+
case "lt":
|
|
991
|
+
return cmp < 0;
|
|
992
|
+
case "gte":
|
|
993
|
+
return cmp >= 0;
|
|
994
|
+
case "lte":
|
|
995
|
+
return cmp <= 0;
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
var FILTER_OPS = {
|
|
999
|
+
eq: (dv, rv) => rv.includes(dv),
|
|
1000
|
+
in: (dv, rv) => rv.includes(dv),
|
|
1001
|
+
neq: (dv, rv) => !rv.includes(dv),
|
|
1002
|
+
gt: (dv, rv) => compareValues(dv, rv[0], "gt"),
|
|
1003
|
+
lt: (dv, rv) => compareValues(dv, rv[0], "lt"),
|
|
1004
|
+
gte: (dv, rv) => compareValues(dv, rv[0], "gte"),
|
|
1005
|
+
lte: (dv, rv) => compareValues(dv, rv[0], "lte")
|
|
1006
|
+
};
|
|
1007
|
+
function filterMatchesDelta(delta, filter, claims) {
|
|
1008
|
+
const col = delta.columns.find((c) => c.column === filter.column);
|
|
1009
|
+
if (!col) {
|
|
1010
|
+
return false;
|
|
1011
|
+
}
|
|
1012
|
+
const deltaValue = String(col.value);
|
|
1013
|
+
const resolvedValues = resolveFilterValue(filter.value, claims);
|
|
1014
|
+
if (resolvedValues.length === 0) {
|
|
1015
|
+
return false;
|
|
1016
|
+
}
|
|
1017
|
+
return FILTER_OPS[filter.op]?.(deltaValue, resolvedValues) ?? false;
|
|
1018
|
+
}
|
|
1019
|
+
function filterDeltas(deltas, context) {
|
|
1020
|
+
if (context.rules.buckets.length === 0) {
|
|
1021
|
+
return deltas;
|
|
1022
|
+
}
|
|
1023
|
+
return deltas.filter(
|
|
1024
|
+
(delta) => context.rules.buckets.some((bucket) => deltaMatchesBucket(delta, bucket, context.claims))
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
function resolveClientBuckets(rules, claims) {
|
|
1028
|
+
return rules.buckets.filter((bucket) => {
|
|
1029
|
+
for (const filter of bucket.filters) {
|
|
1030
|
+
if (filter.value.startsWith("jwt:")) {
|
|
1031
|
+
const resolved = resolveFilterValue(filter.value, claims);
|
|
1032
|
+
if (resolved.length === 0) {
|
|
1033
|
+
return false;
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
return true;
|
|
1038
|
+
}).map((b) => b.name);
|
|
1039
|
+
}
|
|
1040
|
+
function validateSyncRules(config) {
|
|
1041
|
+
if (typeof config !== "object" || config === null) {
|
|
1042
|
+
return Err(new SyncRuleError("Sync rules config must be an object"));
|
|
1043
|
+
}
|
|
1044
|
+
const obj = config;
|
|
1045
|
+
if (typeof obj.version !== "number" || !Number.isInteger(obj.version) || obj.version < 1) {
|
|
1046
|
+
return Err(new SyncRuleError("Sync rules version must be a positive integer"));
|
|
1047
|
+
}
|
|
1048
|
+
if (!Array.isArray(obj.buckets)) {
|
|
1049
|
+
return Err(new SyncRuleError("Sync rules buckets must be an array"));
|
|
1050
|
+
}
|
|
1051
|
+
const seenNames = /* @__PURE__ */ new Set();
|
|
1052
|
+
for (let i = 0; i < obj.buckets.length; i++) {
|
|
1053
|
+
const bucket = obj.buckets[i];
|
|
1054
|
+
if (typeof bucket !== "object" || bucket === null) {
|
|
1055
|
+
return Err(new SyncRuleError(`Bucket at index ${i} must be an object`));
|
|
1056
|
+
}
|
|
1057
|
+
if (typeof bucket.name !== "string" || bucket.name.length === 0) {
|
|
1058
|
+
return Err(new SyncRuleError(`Bucket at index ${i} must have a non-empty name`));
|
|
1059
|
+
}
|
|
1060
|
+
if (seenNames.has(bucket.name)) {
|
|
1061
|
+
return Err(new SyncRuleError(`Duplicate bucket name: "${bucket.name}"`));
|
|
1062
|
+
}
|
|
1063
|
+
seenNames.add(bucket.name);
|
|
1064
|
+
if (!Array.isArray(bucket.tables)) {
|
|
1065
|
+
return Err(new SyncRuleError(`Bucket "${bucket.name}" tables must be an array`));
|
|
1066
|
+
}
|
|
1067
|
+
for (const table of bucket.tables) {
|
|
1068
|
+
if (typeof table !== "string" || table.length === 0) {
|
|
1069
|
+
return Err(
|
|
1070
|
+
new SyncRuleError(`Bucket "${bucket.name}" tables must contain non-empty strings`)
|
|
1071
|
+
);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
if (!Array.isArray(bucket.filters)) {
|
|
1075
|
+
return Err(new SyncRuleError(`Bucket "${bucket.name}" filters must be an array`));
|
|
1076
|
+
}
|
|
1077
|
+
for (let j = 0; j < bucket.filters.length; j++) {
|
|
1078
|
+
const filter = bucket.filters[j];
|
|
1079
|
+
if (typeof filter !== "object" || filter === null) {
|
|
1080
|
+
return Err(
|
|
1081
|
+
new SyncRuleError(`Bucket "${bucket.name}" filter at index ${j} must be an object`)
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
if (typeof filter.column !== "string" || filter.column.length === 0) {
|
|
1085
|
+
return Err(
|
|
1086
|
+
new SyncRuleError(
|
|
1087
|
+
`Bucket "${bucket.name}" filter at index ${j} must have a non-empty column`
|
|
1088
|
+
)
|
|
1089
|
+
);
|
|
1090
|
+
}
|
|
1091
|
+
const validOps = ["eq", "in", "neq", "gt", "lt", "gte", "lte"];
|
|
1092
|
+
if (!validOps.includes(filter.op)) {
|
|
1093
|
+
return Err(
|
|
1094
|
+
new SyncRuleError(
|
|
1095
|
+
`Bucket "${bucket.name}" filter at index ${j} op must be one of: ${validOps.join(", ")}`
|
|
1096
|
+
)
|
|
1097
|
+
);
|
|
1098
|
+
}
|
|
1099
|
+
if (typeof filter.value !== "string" || filter.value.length === 0) {
|
|
1100
|
+
return Err(
|
|
1101
|
+
new SyncRuleError(
|
|
1102
|
+
`Bucket "${bucket.name}" filter at index ${j} must have a non-empty value`
|
|
1103
|
+
)
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
return Ok(void 0);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// ../core/src/validation/identifier.ts
|
|
1112
|
+
var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]{0,63}$/;
|
|
1113
|
+
function isValidIdentifier(name) {
|
|
1114
|
+
return IDENTIFIER_RE.test(name);
|
|
1115
|
+
}
|
|
1116
|
+
function assertValidIdentifier(name) {
|
|
1117
|
+
if (isValidIdentifier(name)) {
|
|
1118
|
+
return Ok(void 0);
|
|
1119
|
+
}
|
|
1120
|
+
return Err(
|
|
1121
|
+
new SchemaError(
|
|
1122
|
+
`Invalid SQL identifier: "${name}". Identifiers must start with a letter or underscore, contain only alphanumeric characters and underscores, and be at most 64 characters long.`
|
|
1123
|
+
)
|
|
1124
|
+
);
|
|
1125
|
+
}
|
|
1126
|
+
function quoteIdentifier(name) {
|
|
1127
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
export {
|
|
1131
|
+
LakeSyncError,
|
|
1132
|
+
ClockDriftError,
|
|
1133
|
+
ConflictError,
|
|
1134
|
+
FlushError,
|
|
1135
|
+
SchemaError,
|
|
1136
|
+
AdapterError,
|
|
1137
|
+
AdapterNotFoundError,
|
|
1138
|
+
BackpressureError,
|
|
1139
|
+
toError,
|
|
1140
|
+
ActionExecutionError,
|
|
1141
|
+
ActionNotSupportedError,
|
|
1142
|
+
ActionValidationError,
|
|
1143
|
+
generateActionId,
|
|
1144
|
+
isActionError,
|
|
1145
|
+
Ok,
|
|
1146
|
+
Err,
|
|
1147
|
+
mapResult,
|
|
1148
|
+
flatMapResult,
|
|
1149
|
+
unwrapOrThrow,
|
|
1150
|
+
fromPromise,
|
|
1151
|
+
validateAction,
|
|
1152
|
+
AuthError,
|
|
1153
|
+
verifyToken,
|
|
1154
|
+
HLC,
|
|
1155
|
+
isIngestTarget,
|
|
1156
|
+
BaseSourcePoller,
|
|
1157
|
+
LWWResolver,
|
|
1158
|
+
resolveLWW,
|
|
1159
|
+
isActionHandler,
|
|
1160
|
+
ConnectorValidationError,
|
|
1161
|
+
CONNECTOR_TYPES,
|
|
1162
|
+
validateConnectorConfig,
|
|
1163
|
+
applyDelta,
|
|
1164
|
+
extractDelta,
|
|
1165
|
+
rowKey,
|
|
1166
|
+
bigintReplacer,
|
|
1167
|
+
bigintReviver,
|
|
1168
|
+
createPassAllRules,
|
|
1169
|
+
createUserScopedRules,
|
|
1170
|
+
SyncRuleError,
|
|
1171
|
+
resolveFilterValue,
|
|
1172
|
+
deltaMatchesBucket,
|
|
1173
|
+
filterDeltas,
|
|
1174
|
+
resolveClientBuckets,
|
|
1175
|
+
validateSyncRules,
|
|
1176
|
+
isValidIdentifier,
|
|
1177
|
+
assertValidIdentifier,
|
|
1178
|
+
quoteIdentifier
|
|
1179
|
+
};
|
|
1180
|
+
//# sourceMappingURL=chunk-ICNT7I3K.js.map
|