lakesync 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.d.ts +185 -20
- package/dist/adapter.js +13 -3
- package/dist/analyst.js +2 -2
- package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Bj9kX9dv.d.ts} +76 -19
- package/dist/catalogue.d.ts +1 -1
- package/dist/catalogue.js +3 -3
- package/dist/chunk-DGUM43GV.js +11 -0
- package/dist/{chunk-IRJ4QRWV.js → chunk-JI4C4R5H.js} +249 -140
- package/dist/chunk-JI4C4R5H.js.map +1 -0
- package/dist/{chunk-FHVTUKXL.js → chunk-KVSWLIJR.js} +2 -2
- package/dist/{chunk-P3FT7QCW.js → chunk-LDFFCG2K.js} +377 -247
- package/dist/chunk-LDFFCG2K.js.map +1 -0
- package/dist/{chunk-GUJWMK5P.js → chunk-LPWXOYNS.js} +373 -350
- package/dist/chunk-LPWXOYNS.js.map +1 -0
- package/dist/{chunk-QMS7TGFL.js → chunk-PYRS74YP.js} +15 -4
- package/dist/{chunk-QMS7TGFL.js.map → chunk-PYRS74YP.js.map} +1 -1
- package/dist/{chunk-NCZYFZ3B.js → chunk-QNITY4F6.js} +30 -7
- package/dist/{chunk-NCZYFZ3B.js.map → chunk-QNITY4F6.js.map} +1 -1
- package/dist/{chunk-SF7Y6ZUA.js → chunk-SSICS5KI.js} +2 -2
- package/dist/{chunk-UAUQGP3B.js → chunk-TMLG32QV.js} +2 -2
- package/dist/client.d.ts +164 -13
- package/dist/client.js +310 -163
- package/dist/client.js.map +1 -1
- package/dist/compactor.d.ts +1 -1
- package/dist/compactor.js +4 -4
- package/dist/connector-jira.d.ts +2 -2
- package/dist/connector-jira.js +3 -3
- package/dist/connector-salesforce.d.ts +2 -2
- package/dist/connector-salesforce.js +3 -3
- package/dist/{coordinator-D32a5rNk.d.ts → coordinator-NXy6tA0h.d.ts} +23 -16
- package/dist/{db-types-BlN-4KbQ.d.ts → db-types-CfLMUBfW.d.ts} +1 -1
- package/dist/gateway-server.d.ts +158 -64
- package/dist/gateway-server.js +482 -4003
- package/dist/gateway-server.js.map +1 -1
- package/dist/gateway.d.ts +61 -104
- package/dist/gateway.js +12 -6
- package/dist/index.d.ts +45 -10
- package/dist/index.js +14 -2
- package/dist/parquet.d.ts +1 -1
- package/dist/parquet.js +3 -3
- package/dist/proto.d.ts +1 -1
- package/dist/proto.js +3 -3
- package/dist/react.d.ts +47 -10
- package/dist/react.js +88 -40
- package/dist/react.js.map +1 -1
- package/dist/{registry-CPTgO9jv.d.ts → registry-BcspAtZI.d.ts} +19 -4
- package/dist/{gateway-Bpvatd9n.d.ts → request-handler-pUvL7ozF.d.ts} +139 -10
- package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
- package/dist/{src-FPJQYQNA.js → src-B6NLV3FP.js} +4 -4
- package/dist/{src-RHKJFQKR.js → src-ROW4XLO7.js} +15 -3
- package/dist/{src-CLCALYDT.js → src-ZRHKG42A.js} +4 -4
- package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
- package/dist/{types-D-E0VrfS.d.ts → types-BrcD1oJg.d.ts} +26 -19
- package/package.json +1 -1
- package/dist/chunk-7D4SUZUM.js +0 -38
- package/dist/chunk-GUJWMK5P.js.map +0 -1
- package/dist/chunk-IRJ4QRWV.js.map +0 -1
- package/dist/chunk-P3FT7QCW.js.map +0 -1
- /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
- /package/dist/{chunk-FHVTUKXL.js.map → chunk-KVSWLIJR.js.map} +0 -0
- /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-SSICS5KI.js.map} +0 -0
- /package/dist/{chunk-UAUQGP3B.js.map → chunk-TMLG32QV.js.map} +0 -0
- /package/dist/{src-CLCALYDT.js.map → src-B6NLV3FP.js.map} +0 -0
- /package/dist/{src-FPJQYQNA.js.map → src-ROW4XLO7.js.map} +0 -0
- /package/dist/{src-RHKJFQKR.js.map → src-ZRHKG42A.js.map} +0 -0
|
@@ -387,88 +387,45 @@ var HLC = class _HLC {
|
|
|
387
387
|
}
|
|
388
388
|
};
|
|
389
389
|
|
|
390
|
-
// ../core/src/
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
}
|
|
394
|
-
var DEFAULT_CHUNK_SIZE = 500;
|
|
395
|
-
var DEFAULT_FLUSH_THRESHOLD = 0.7;
|
|
396
|
-
var BaseSourcePoller = class {
|
|
397
|
-
gateway;
|
|
398
|
-
hlc;
|
|
390
|
+
// ../core/src/polling/chunked-pusher.ts
|
|
391
|
+
var ChunkedPusher = class {
|
|
392
|
+
target;
|
|
399
393
|
clientId;
|
|
400
|
-
intervalMs;
|
|
401
|
-
timer = null;
|
|
402
|
-
running = false;
|
|
403
394
|
chunkSize;
|
|
404
|
-
|
|
405
|
-
flushThreshold;
|
|
395
|
+
pressure;
|
|
406
396
|
pendingDeltas = [];
|
|
407
397
|
constructor(config) {
|
|
408
|
-
this.
|
|
409
|
-
this.
|
|
410
|
-
this.
|
|
411
|
-
this.
|
|
412
|
-
this.chunkSize = config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE;
|
|
413
|
-
this.memoryBudgetBytes = config.memory?.memoryBudgetBytes;
|
|
414
|
-
this.flushThreshold = config.memory?.flushThreshold ?? DEFAULT_FLUSH_THRESHOLD;
|
|
415
|
-
}
|
|
416
|
-
/** Start the polling loop. */
|
|
417
|
-
start() {
|
|
418
|
-
if (this.running) return;
|
|
419
|
-
this.running = true;
|
|
420
|
-
this.schedulePoll();
|
|
421
|
-
}
|
|
422
|
-
/** Stop the polling loop. */
|
|
423
|
-
stop() {
|
|
424
|
-
this.running = false;
|
|
425
|
-
if (this.timer) {
|
|
426
|
-
clearTimeout(this.timer);
|
|
427
|
-
this.timer = null;
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
/** Whether the poller is currently running. */
|
|
431
|
-
get isRunning() {
|
|
432
|
-
return this.running;
|
|
433
|
-
}
|
|
434
|
-
/**
|
|
435
|
-
* Execute a single poll cycle without the timer loop.
|
|
436
|
-
* Convenience for serverless consumers who trigger polls manually.
|
|
437
|
-
*/
|
|
438
|
-
async pollOnce() {
|
|
439
|
-
return this.poll();
|
|
440
|
-
}
|
|
441
|
-
/** Push collected deltas to the gateway (single-shot, backward compat). */
|
|
442
|
-
pushDeltas(deltas) {
|
|
443
|
-
if (deltas.length === 0) return;
|
|
444
|
-
const push = {
|
|
445
|
-
clientId: this.clientId,
|
|
446
|
-
deltas,
|
|
447
|
-
lastSeenHlc: 0n
|
|
448
|
-
};
|
|
449
|
-
this.gateway.handlePush(push);
|
|
398
|
+
this.target = config.target;
|
|
399
|
+
this.clientId = config.clientId;
|
|
400
|
+
this.chunkSize = config.chunkSize;
|
|
401
|
+
this.pressure = config.pressure;
|
|
450
402
|
}
|
|
451
403
|
/**
|
|
452
404
|
* Accumulate a single delta. When `chunkSize` is reached, the pending
|
|
453
405
|
* deltas are automatically pushed (and flushed if needed).
|
|
454
406
|
*/
|
|
455
|
-
async
|
|
407
|
+
async accumulate(delta) {
|
|
456
408
|
this.pendingDeltas.push(delta);
|
|
457
409
|
if (this.pendingDeltas.length >= this.chunkSize) {
|
|
458
410
|
await this.pushPendingChunk();
|
|
459
411
|
}
|
|
460
412
|
}
|
|
461
|
-
/** Flush any remaining accumulated deltas.
|
|
462
|
-
async
|
|
413
|
+
/** Flush any remaining accumulated deltas. */
|
|
414
|
+
async flush() {
|
|
463
415
|
if (this.pendingDeltas.length > 0) {
|
|
464
416
|
await this.pushPendingChunk();
|
|
465
417
|
}
|
|
466
418
|
}
|
|
467
|
-
/**
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
419
|
+
/** Push deltas directly (single-shot, backward compat). */
|
|
420
|
+
pushImmediate(deltas) {
|
|
421
|
+
if (deltas.length === 0) return;
|
|
422
|
+
const push = {
|
|
423
|
+
clientId: this.clientId,
|
|
424
|
+
deltas,
|
|
425
|
+
lastSeenHlc: 0n
|
|
426
|
+
};
|
|
427
|
+
this.target.handlePush(push);
|
|
428
|
+
}
|
|
472
429
|
async pushPendingChunk() {
|
|
473
430
|
const chunk = this.pendingDeltas;
|
|
474
431
|
this.pendingDeltas = [];
|
|
@@ -476,45 +433,162 @@ var BaseSourcePoller = class {
|
|
|
476
433
|
}
|
|
477
434
|
async pushChunkWithFlush(chunk) {
|
|
478
435
|
if (chunk.length === 0) return;
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
if (this.shouldFlushTarget(target)) {
|
|
482
|
-
await target.flush();
|
|
483
|
-
}
|
|
436
|
+
if (this.pressure) {
|
|
437
|
+
await this.pressure.checkAndFlush();
|
|
484
438
|
}
|
|
485
439
|
const push = {
|
|
486
440
|
clientId: this.clientId,
|
|
487
441
|
deltas: chunk,
|
|
488
442
|
lastSeenHlc: 0n
|
|
489
443
|
};
|
|
490
|
-
const result = target.handlePush(push);
|
|
444
|
+
const result = this.target.handlePush(push);
|
|
491
445
|
if (result && typeof result === "object" && "ok" in result && !result.ok) {
|
|
492
|
-
if (
|
|
493
|
-
await
|
|
494
|
-
target.handlePush(push);
|
|
446
|
+
if (this.pressure) {
|
|
447
|
+
await this.pressure.forceFlush();
|
|
448
|
+
this.target.handlePush(push);
|
|
495
449
|
}
|
|
496
450
|
}
|
|
497
451
|
}
|
|
498
|
-
|
|
499
|
-
|
|
452
|
+
};
|
|
453
|
+
|
|
454
|
+
// ../core/src/polling/pressure-manager.ts
|
|
455
|
+
var PressureManager = class {
|
|
456
|
+
target;
|
|
457
|
+
memoryBudgetBytes;
|
|
458
|
+
flushThreshold;
|
|
459
|
+
constructor(config) {
|
|
460
|
+
this.target = config.target;
|
|
461
|
+
this.memoryBudgetBytes = config.memoryBudgetBytes;
|
|
462
|
+
this.flushThreshold = config.flushThreshold ?? 0.7;
|
|
463
|
+
}
|
|
464
|
+
/** Check buffer pressure and flush if thresholds are exceeded. */
|
|
465
|
+
async checkAndFlush() {
|
|
466
|
+
if (this.shouldFlush()) {
|
|
467
|
+
await this.target.flush();
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
/** Force a flush regardless of current pressure. */
|
|
471
|
+
async forceFlush() {
|
|
472
|
+
await this.target.flush();
|
|
473
|
+
}
|
|
474
|
+
shouldFlush() {
|
|
475
|
+
if (this.target.shouldFlush()) return true;
|
|
500
476
|
if (this.memoryBudgetBytes != null) {
|
|
501
477
|
const threshold = Math.floor(this.memoryBudgetBytes * this.flushThreshold);
|
|
502
|
-
if (target.bufferStats.byteSize >= threshold) return true;
|
|
478
|
+
if (this.target.bufferStats.byteSize >= threshold) return true;
|
|
503
479
|
}
|
|
504
480
|
return false;
|
|
505
481
|
}
|
|
506
|
-
|
|
482
|
+
};
|
|
483
|
+
|
|
484
|
+
// ../core/src/polling/scheduler.ts
|
|
485
|
+
var PollingScheduler = class {
|
|
486
|
+
pollFn;
|
|
487
|
+
intervalMs;
|
|
488
|
+
timer = null;
|
|
489
|
+
running = false;
|
|
490
|
+
constructor(pollFn, intervalMs) {
|
|
491
|
+
this.pollFn = pollFn;
|
|
492
|
+
this.intervalMs = intervalMs;
|
|
493
|
+
}
|
|
494
|
+
/** Start the polling loop. No-op if already running. */
|
|
495
|
+
start() {
|
|
496
|
+
if (this.running) return;
|
|
497
|
+
this.running = true;
|
|
498
|
+
this.schedule();
|
|
499
|
+
}
|
|
500
|
+
/** Stop the polling loop. */
|
|
501
|
+
stop() {
|
|
502
|
+
this.running = false;
|
|
503
|
+
if (this.timer) {
|
|
504
|
+
clearTimeout(this.timer);
|
|
505
|
+
this.timer = null;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
/** Whether the scheduler is currently running. */
|
|
509
|
+
get isRunning() {
|
|
510
|
+
return this.running;
|
|
511
|
+
}
|
|
512
|
+
/** Execute a single poll cycle without the timer loop. */
|
|
513
|
+
async pollOnce() {
|
|
514
|
+
return this.pollFn();
|
|
515
|
+
}
|
|
516
|
+
schedule() {
|
|
507
517
|
if (!this.running) return;
|
|
508
518
|
this.timer = setTimeout(async () => {
|
|
509
519
|
try {
|
|
510
|
-
await this.
|
|
520
|
+
await this.pollFn();
|
|
511
521
|
} catch {
|
|
512
522
|
}
|
|
513
|
-
this.
|
|
523
|
+
this.schedule();
|
|
514
524
|
}, this.intervalMs);
|
|
515
525
|
}
|
|
516
526
|
};
|
|
517
527
|
|
|
528
|
+
// ../core/src/base-poller.ts
|
|
529
|
+
function isIngestTarget(target) {
|
|
530
|
+
return typeof target.flush === "function" && typeof target.shouldFlush === "function" && "bufferStats" in target;
|
|
531
|
+
}
|
|
532
|
+
var DEFAULT_CHUNK_SIZE = 500;
|
|
533
|
+
var BaseSourcePoller = class {
|
|
534
|
+
gateway;
|
|
535
|
+
hlc;
|
|
536
|
+
clientId;
|
|
537
|
+
scheduler;
|
|
538
|
+
pusher;
|
|
539
|
+
constructor(config) {
|
|
540
|
+
this.gateway = config.gateway;
|
|
541
|
+
this.hlc = new HLC();
|
|
542
|
+
this.clientId = `ingest:${config.name}`;
|
|
543
|
+
const pressure = isIngestTarget(config.gateway) ? new PressureManager({
|
|
544
|
+
target: config.gateway,
|
|
545
|
+
memoryBudgetBytes: config.memory?.memoryBudgetBytes,
|
|
546
|
+
flushThreshold: config.memory?.flushThreshold
|
|
547
|
+
}) : null;
|
|
548
|
+
this.pusher = new ChunkedPusher({
|
|
549
|
+
target: config.gateway,
|
|
550
|
+
clientId: this.clientId,
|
|
551
|
+
chunkSize: config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE,
|
|
552
|
+
pressure
|
|
553
|
+
});
|
|
554
|
+
this.scheduler = new PollingScheduler(() => this.poll(), config.intervalMs);
|
|
555
|
+
}
|
|
556
|
+
/** Start the polling loop. */
|
|
557
|
+
start() {
|
|
558
|
+
this.scheduler.start();
|
|
559
|
+
}
|
|
560
|
+
/** Stop the polling loop. */
|
|
561
|
+
stop() {
|
|
562
|
+
this.scheduler.stop();
|
|
563
|
+
}
|
|
564
|
+
/** Whether the poller is currently running. */
|
|
565
|
+
get isRunning() {
|
|
566
|
+
return this.scheduler.isRunning;
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Execute a single poll cycle without the timer loop.
|
|
570
|
+
* Convenience for serverless consumers who trigger polls manually.
|
|
571
|
+
*/
|
|
572
|
+
async pollOnce() {
|
|
573
|
+
return this.scheduler.pollOnce();
|
|
574
|
+
}
|
|
575
|
+
/** Push collected deltas to the gateway (single-shot, backward compat). */
|
|
576
|
+
pushDeltas(deltas) {
|
|
577
|
+
this.pusher.pushImmediate(deltas);
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Accumulate a single delta. When `chunkSize` is reached, the pending
|
|
581
|
+
* deltas are automatically pushed (and flushed if needed).
|
|
582
|
+
*/
|
|
583
|
+
async accumulateDelta(delta) {
|
|
584
|
+
await this.pusher.accumulate(delta);
|
|
585
|
+
}
|
|
586
|
+
/** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
|
|
587
|
+
async flushAccumulator() {
|
|
588
|
+
await this.pusher.flush();
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
|
|
518
592
|
// ../core/src/callback-push-target.ts
|
|
519
593
|
var CallbackPushTarget = class {
|
|
520
594
|
onPush;
|
|
@@ -621,6 +695,35 @@ var ConnectorValidationError = class extends LakeSyncError {
|
|
|
621
695
|
};
|
|
622
696
|
|
|
623
697
|
// ../core/src/connector/registry.ts
|
|
698
|
+
function createConnectorRegistry(descriptors2) {
|
|
699
|
+
const map = /* @__PURE__ */ new Map();
|
|
700
|
+
for (const d of descriptors2) {
|
|
701
|
+
map.set(d.type, d);
|
|
702
|
+
}
|
|
703
|
+
return buildRegistry(map);
|
|
704
|
+
}
|
|
705
|
+
function buildRegistry(map) {
|
|
706
|
+
return {
|
|
707
|
+
get(type) {
|
|
708
|
+
return map.get(type);
|
|
709
|
+
},
|
|
710
|
+
list() {
|
|
711
|
+
return [...map.values()].sort((a, b) => a.type.localeCompare(b.type));
|
|
712
|
+
},
|
|
713
|
+
with(descriptor) {
|
|
714
|
+
const next = new Map(map);
|
|
715
|
+
next.set(descriptor.type, descriptor);
|
|
716
|
+
return buildRegistry(next);
|
|
717
|
+
},
|
|
718
|
+
withOutputSchemas(type, schemas) {
|
|
719
|
+
const existing = map.get(type);
|
|
720
|
+
if (!existing) return buildRegistry(map);
|
|
721
|
+
const next = new Map(map);
|
|
722
|
+
next.set(type, { ...existing, outputTables: schemas });
|
|
723
|
+
return buildRegistry(next);
|
|
724
|
+
}
|
|
725
|
+
};
|
|
726
|
+
}
|
|
624
727
|
var descriptors = /* @__PURE__ */ new Map();
|
|
625
728
|
function registerConnectorDescriptor(descriptor) {
|
|
626
729
|
descriptors.set(descriptor.type, descriptor);
|
|
@@ -642,6 +745,162 @@ var CONNECTOR_TYPES = ["postgres", "mysql", "bigquery", "jira", "salesforce"];
|
|
|
642
745
|
|
|
643
746
|
// ../core/src/connector/validate.ts
|
|
644
747
|
var VALID_STRATEGIES = /* @__PURE__ */ new Set(["cursor", "diff"]);
|
|
748
|
+
function validatePostgresConfig(obj) {
|
|
749
|
+
const pg = obj.postgres;
|
|
750
|
+
if (typeof pg !== "object" || pg === null) {
|
|
751
|
+
return Err(
|
|
752
|
+
new ConnectorValidationError('Connector type "postgres" requires a postgres config object')
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
const pgObj = pg;
|
|
756
|
+
if (typeof pgObj.connectionString !== "string" || pgObj.connectionString.length === 0) {
|
|
757
|
+
return Err(
|
|
758
|
+
new ConnectorValidationError("Postgres connector requires a non-empty connectionString")
|
|
759
|
+
);
|
|
760
|
+
}
|
|
761
|
+
return Ok(void 0);
|
|
762
|
+
}
|
|
763
|
+
function validateMySQLConfig(obj) {
|
|
764
|
+
const my = obj.mysql;
|
|
765
|
+
if (typeof my !== "object" || my === null) {
|
|
766
|
+
return Err(
|
|
767
|
+
new ConnectorValidationError('Connector type "mysql" requires a mysql config object')
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
const myObj = my;
|
|
771
|
+
if (typeof myObj.connectionString !== "string" || myObj.connectionString.length === 0) {
|
|
772
|
+
return Err(
|
|
773
|
+
new ConnectorValidationError("MySQL connector requires a non-empty connectionString")
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
return Ok(void 0);
|
|
777
|
+
}
|
|
778
|
+
function validateBigQueryConfig(obj) {
|
|
779
|
+
const bq = obj.bigquery;
|
|
780
|
+
if (typeof bq !== "object" || bq === null) {
|
|
781
|
+
return Err(
|
|
782
|
+
new ConnectorValidationError('Connector type "bigquery" requires a bigquery config object')
|
|
783
|
+
);
|
|
784
|
+
}
|
|
785
|
+
const bqObj = bq;
|
|
786
|
+
if (typeof bqObj.projectId !== "string" || bqObj.projectId.length === 0) {
|
|
787
|
+
return Err(new ConnectorValidationError("BigQuery connector requires a non-empty projectId"));
|
|
788
|
+
}
|
|
789
|
+
if (typeof bqObj.dataset !== "string" || bqObj.dataset.length === 0) {
|
|
790
|
+
return Err(new ConnectorValidationError("BigQuery connector requires a non-empty dataset"));
|
|
791
|
+
}
|
|
792
|
+
return Ok(void 0);
|
|
793
|
+
}
|
|
794
|
+
function validateJiraConfig(obj) {
|
|
795
|
+
const jira = obj.jira;
|
|
796
|
+
if (typeof jira !== "object" || jira === null) {
|
|
797
|
+
return Err(new ConnectorValidationError('Connector type "jira" requires a jira config object'));
|
|
798
|
+
}
|
|
799
|
+
const jiraObj = jira;
|
|
800
|
+
if (typeof jiraObj.domain !== "string" || jiraObj.domain.length === 0) {
|
|
801
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty domain"));
|
|
802
|
+
}
|
|
803
|
+
if (typeof jiraObj.email !== "string" || jiraObj.email.length === 0) {
|
|
804
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty email"));
|
|
805
|
+
}
|
|
806
|
+
if (typeof jiraObj.apiToken !== "string" || jiraObj.apiToken.length === 0) {
|
|
807
|
+
return Err(new ConnectorValidationError("Jira connector requires a non-empty apiToken"));
|
|
808
|
+
}
|
|
809
|
+
return Ok(void 0);
|
|
810
|
+
}
|
|
811
|
+
function validateSalesforceConfig(obj) {
|
|
812
|
+
const sf = obj.salesforce;
|
|
813
|
+
if (typeof sf !== "object" || sf === null) {
|
|
814
|
+
return Err(
|
|
815
|
+
new ConnectorValidationError(
|
|
816
|
+
'Connector type "salesforce" requires a salesforce config object'
|
|
817
|
+
)
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
const sfObj = sf;
|
|
821
|
+
if (typeof sfObj.instanceUrl !== "string" || sfObj.instanceUrl.length === 0) {
|
|
822
|
+
return Err(
|
|
823
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty instanceUrl")
|
|
824
|
+
);
|
|
825
|
+
}
|
|
826
|
+
if (typeof sfObj.clientId !== "string" || sfObj.clientId.length === 0) {
|
|
827
|
+
return Err(new ConnectorValidationError("Salesforce connector requires a non-empty clientId"));
|
|
828
|
+
}
|
|
829
|
+
if (typeof sfObj.clientSecret !== "string" || sfObj.clientSecret.length === 0) {
|
|
830
|
+
return Err(
|
|
831
|
+
new ConnectorValidationError("Salesforce connector requires a non-empty clientSecret")
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
if (typeof sfObj.username !== "string" || sfObj.username.length === 0) {
|
|
835
|
+
return Err(new ConnectorValidationError("Salesforce connector requires a non-empty username"));
|
|
836
|
+
}
|
|
837
|
+
if (typeof sfObj.password !== "string" || sfObj.password.length === 0) {
|
|
838
|
+
return Err(new ConnectorValidationError("Salesforce connector requires a non-empty password"));
|
|
839
|
+
}
|
|
840
|
+
return Ok(void 0);
|
|
841
|
+
}
|
|
842
|
+
function validateIngestConfig(obj, connectorType) {
|
|
843
|
+
if (obj.ingest === void 0) return Ok(void 0);
|
|
844
|
+
if (typeof obj.ingest !== "object" || obj.ingest === null) {
|
|
845
|
+
return Err(new ConnectorValidationError("Ingest config must be an object"));
|
|
846
|
+
}
|
|
847
|
+
const ingest = obj.ingest;
|
|
848
|
+
if (connectorType === "jira" || connectorType === "salesforce") {
|
|
849
|
+
if (ingest.intervalMs !== void 0) {
|
|
850
|
+
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
851
|
+
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
return Ok(void 0);
|
|
855
|
+
}
|
|
856
|
+
if (!Array.isArray(ingest.tables) || ingest.tables.length === 0) {
|
|
857
|
+
return Err(new ConnectorValidationError("Ingest config must have a non-empty tables array"));
|
|
858
|
+
}
|
|
859
|
+
for (let i = 0; i < ingest.tables.length; i++) {
|
|
860
|
+
const table = ingest.tables[i];
|
|
861
|
+
if (typeof table !== "object" || table === null) {
|
|
862
|
+
return Err(new ConnectorValidationError(`Ingest table at index ${i} must be an object`));
|
|
863
|
+
}
|
|
864
|
+
if (typeof table.table !== "string" || table.table.length === 0) {
|
|
865
|
+
return Err(
|
|
866
|
+
new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty table name`)
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
if (typeof table.query !== "string" || table.query.length === 0) {
|
|
870
|
+
return Err(
|
|
871
|
+
new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty query`)
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
if (typeof table.strategy !== "object" || table.strategy === null) {
|
|
875
|
+
return Err(
|
|
876
|
+
new ConnectorValidationError(`Ingest table at index ${i} must have a strategy object`)
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
const strategy = table.strategy;
|
|
880
|
+
if (!VALID_STRATEGIES.has(strategy.type)) {
|
|
881
|
+
return Err(
|
|
882
|
+
new ConnectorValidationError(
|
|
883
|
+
`Ingest table at index ${i} strategy type must be "cursor" or "diff"`
|
|
884
|
+
)
|
|
885
|
+
);
|
|
886
|
+
}
|
|
887
|
+
if (strategy.type === "cursor") {
|
|
888
|
+
if (typeof strategy.cursorColumn !== "string" || strategy.cursorColumn.length === 0) {
|
|
889
|
+
return Err(
|
|
890
|
+
new ConnectorValidationError(
|
|
891
|
+
`Ingest table at index ${i} cursor strategy requires a non-empty cursorColumn`
|
|
892
|
+
)
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (ingest.intervalMs !== void 0) {
|
|
898
|
+
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
899
|
+
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
return Ok(void 0);
|
|
903
|
+
}
|
|
645
904
|
function validateConnectorConfig(input) {
|
|
646
905
|
if (typeof input !== "object" || input === null) {
|
|
647
906
|
return Err(new ConnectorValidationError("Connector config must be an object"));
|
|
@@ -656,178 +915,27 @@ function validateConnectorConfig(input) {
|
|
|
656
915
|
);
|
|
657
916
|
}
|
|
658
917
|
const connectorType = obj.type;
|
|
918
|
+
let typeResult;
|
|
659
919
|
switch (connectorType) {
|
|
660
|
-
case "postgres":
|
|
661
|
-
|
|
662
|
-
if (typeof pg !== "object" || pg === null) {
|
|
663
|
-
return Err(
|
|
664
|
-
new ConnectorValidationError(
|
|
665
|
-
'Connector type "postgres" requires a postgres config object'
|
|
666
|
-
)
|
|
667
|
-
);
|
|
668
|
-
}
|
|
669
|
-
const pgObj = pg;
|
|
670
|
-
if (typeof pgObj.connectionString !== "string" || pgObj.connectionString.length === 0) {
|
|
671
|
-
return Err(
|
|
672
|
-
new ConnectorValidationError("Postgres connector requires a non-empty connectionString")
|
|
673
|
-
);
|
|
674
|
-
}
|
|
920
|
+
case "postgres":
|
|
921
|
+
typeResult = validatePostgresConfig(obj);
|
|
675
922
|
break;
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
const my = obj.mysql;
|
|
679
|
-
if (typeof my !== "object" || my === null) {
|
|
680
|
-
return Err(
|
|
681
|
-
new ConnectorValidationError('Connector type "mysql" requires a mysql config object')
|
|
682
|
-
);
|
|
683
|
-
}
|
|
684
|
-
const myObj = my;
|
|
685
|
-
if (typeof myObj.connectionString !== "string" || myObj.connectionString.length === 0) {
|
|
686
|
-
return Err(
|
|
687
|
-
new ConnectorValidationError("MySQL connector requires a non-empty connectionString")
|
|
688
|
-
);
|
|
689
|
-
}
|
|
923
|
+
case "mysql":
|
|
924
|
+
typeResult = validateMySQLConfig(obj);
|
|
690
925
|
break;
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
const bq = obj.bigquery;
|
|
694
|
-
if (typeof bq !== "object" || bq === null) {
|
|
695
|
-
return Err(
|
|
696
|
-
new ConnectorValidationError(
|
|
697
|
-
'Connector type "bigquery" requires a bigquery config object'
|
|
698
|
-
)
|
|
699
|
-
);
|
|
700
|
-
}
|
|
701
|
-
const bqObj = bq;
|
|
702
|
-
if (typeof bqObj.projectId !== "string" || bqObj.projectId.length === 0) {
|
|
703
|
-
return Err(
|
|
704
|
-
new ConnectorValidationError("BigQuery connector requires a non-empty projectId")
|
|
705
|
-
);
|
|
706
|
-
}
|
|
707
|
-
if (typeof bqObj.dataset !== "string" || bqObj.dataset.length === 0) {
|
|
708
|
-
return Err(new ConnectorValidationError("BigQuery connector requires a non-empty dataset"));
|
|
709
|
-
}
|
|
926
|
+
case "bigquery":
|
|
927
|
+
typeResult = validateBigQueryConfig(obj);
|
|
710
928
|
break;
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
const jira = obj.jira;
|
|
714
|
-
if (typeof jira !== "object" || jira === null) {
|
|
715
|
-
return Err(
|
|
716
|
-
new ConnectorValidationError('Connector type "jira" requires a jira config object')
|
|
717
|
-
);
|
|
718
|
-
}
|
|
719
|
-
const jiraObj = jira;
|
|
720
|
-
if (typeof jiraObj.domain !== "string" || jiraObj.domain.length === 0) {
|
|
721
|
-
return Err(new ConnectorValidationError("Jira connector requires a non-empty domain"));
|
|
722
|
-
}
|
|
723
|
-
if (typeof jiraObj.email !== "string" || jiraObj.email.length === 0) {
|
|
724
|
-
return Err(new ConnectorValidationError("Jira connector requires a non-empty email"));
|
|
725
|
-
}
|
|
726
|
-
if (typeof jiraObj.apiToken !== "string" || jiraObj.apiToken.length === 0) {
|
|
727
|
-
return Err(new ConnectorValidationError("Jira connector requires a non-empty apiToken"));
|
|
728
|
-
}
|
|
929
|
+
case "jira":
|
|
930
|
+
typeResult = validateJiraConfig(obj);
|
|
729
931
|
break;
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
const sf = obj.salesforce;
|
|
733
|
-
if (typeof sf !== "object" || sf === null) {
|
|
734
|
-
return Err(
|
|
735
|
-
new ConnectorValidationError(
|
|
736
|
-
'Connector type "salesforce" requires a salesforce config object'
|
|
737
|
-
)
|
|
738
|
-
);
|
|
739
|
-
}
|
|
740
|
-
const sfObj = sf;
|
|
741
|
-
if (typeof sfObj.instanceUrl !== "string" || sfObj.instanceUrl.length === 0) {
|
|
742
|
-
return Err(
|
|
743
|
-
new ConnectorValidationError("Salesforce connector requires a non-empty instanceUrl")
|
|
744
|
-
);
|
|
745
|
-
}
|
|
746
|
-
if (typeof sfObj.clientId !== "string" || sfObj.clientId.length === 0) {
|
|
747
|
-
return Err(
|
|
748
|
-
new ConnectorValidationError("Salesforce connector requires a non-empty clientId")
|
|
749
|
-
);
|
|
750
|
-
}
|
|
751
|
-
if (typeof sfObj.clientSecret !== "string" || sfObj.clientSecret.length === 0) {
|
|
752
|
-
return Err(
|
|
753
|
-
new ConnectorValidationError("Salesforce connector requires a non-empty clientSecret")
|
|
754
|
-
);
|
|
755
|
-
}
|
|
756
|
-
if (typeof sfObj.username !== "string" || sfObj.username.length === 0) {
|
|
757
|
-
return Err(
|
|
758
|
-
new ConnectorValidationError("Salesforce connector requires a non-empty username")
|
|
759
|
-
);
|
|
760
|
-
}
|
|
761
|
-
if (typeof sfObj.password !== "string" || sfObj.password.length === 0) {
|
|
762
|
-
return Err(
|
|
763
|
-
new ConnectorValidationError("Salesforce connector requires a non-empty password")
|
|
764
|
-
);
|
|
765
|
-
}
|
|
932
|
+
case "salesforce":
|
|
933
|
+
typeResult = validateSalesforceConfig(obj);
|
|
766
934
|
break;
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
if (obj.ingest !== void 0) {
|
|
770
|
-
if (typeof obj.ingest !== "object" || obj.ingest === null) {
|
|
771
|
-
return Err(new ConnectorValidationError("Ingest config must be an object"));
|
|
772
|
-
}
|
|
773
|
-
const ingest = obj.ingest;
|
|
774
|
-
if (connectorType === "jira" || connectorType === "salesforce") {
|
|
775
|
-
if (ingest.intervalMs !== void 0) {
|
|
776
|
-
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
777
|
-
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
return Ok(input);
|
|
781
|
-
}
|
|
782
|
-
if (!Array.isArray(ingest.tables) || ingest.tables.length === 0) {
|
|
783
|
-
return Err(new ConnectorValidationError("Ingest config must have a non-empty tables array"));
|
|
784
|
-
}
|
|
785
|
-
for (let i = 0; i < ingest.tables.length; i++) {
|
|
786
|
-
const table = ingest.tables[i];
|
|
787
|
-
if (typeof table !== "object" || table === null) {
|
|
788
|
-
return Err(new ConnectorValidationError(`Ingest table at index ${i} must be an object`));
|
|
789
|
-
}
|
|
790
|
-
if (typeof table.table !== "string" || table.table.length === 0) {
|
|
791
|
-
return Err(
|
|
792
|
-
new ConnectorValidationError(
|
|
793
|
-
`Ingest table at index ${i} must have a non-empty table name`
|
|
794
|
-
)
|
|
795
|
-
);
|
|
796
|
-
}
|
|
797
|
-
if (typeof table.query !== "string" || table.query.length === 0) {
|
|
798
|
-
return Err(
|
|
799
|
-
new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty query`)
|
|
800
|
-
);
|
|
801
|
-
}
|
|
802
|
-
if (typeof table.strategy !== "object" || table.strategy === null) {
|
|
803
|
-
return Err(
|
|
804
|
-
new ConnectorValidationError(`Ingest table at index ${i} must have a strategy object`)
|
|
805
|
-
);
|
|
806
|
-
}
|
|
807
|
-
const strategy = table.strategy;
|
|
808
|
-
if (!VALID_STRATEGIES.has(strategy.type)) {
|
|
809
|
-
return Err(
|
|
810
|
-
new ConnectorValidationError(
|
|
811
|
-
`Ingest table at index ${i} strategy type must be "cursor" or "diff"`
|
|
812
|
-
)
|
|
813
|
-
);
|
|
814
|
-
}
|
|
815
|
-
if (strategy.type === "cursor") {
|
|
816
|
-
if (typeof strategy.cursorColumn !== "string" || strategy.cursorColumn.length === 0) {
|
|
817
|
-
return Err(
|
|
818
|
-
new ConnectorValidationError(
|
|
819
|
-
`Ingest table at index ${i} cursor strategy requires a non-empty cursorColumn`
|
|
820
|
-
)
|
|
821
|
-
);
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
if (ingest.intervalMs !== void 0) {
|
|
826
|
-
if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
|
|
827
|
-
return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
935
|
}
|
|
936
|
+
if (!typeResult.ok) return typeResult;
|
|
937
|
+
const ingestResult = validateIngestConfig(obj, connectorType);
|
|
938
|
+
if (!ingestResult.ok) return ingestResult;
|
|
831
939
|
return Ok(input);
|
|
832
940
|
}
|
|
833
941
|
|
|
@@ -1105,12 +1213,27 @@ registerConnectorDescriptor({
|
|
|
1105
1213
|
});
|
|
1106
1214
|
|
|
1107
1215
|
// ../core/src/create-poller.ts
|
|
1216
|
+
function createPollerRegistry(factories = /* @__PURE__ */ new Map()) {
|
|
1217
|
+
return buildPollerRegistry(new Map(factories));
|
|
1218
|
+
}
|
|
1219
|
+
function buildPollerRegistry(map) {
|
|
1220
|
+
return {
|
|
1221
|
+
get(type) {
|
|
1222
|
+
return map.get(type);
|
|
1223
|
+
},
|
|
1224
|
+
with(type, factory) {
|
|
1225
|
+
const next = new Map(map);
|
|
1226
|
+
next.set(type, factory);
|
|
1227
|
+
return buildPollerRegistry(next);
|
|
1228
|
+
}
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1108
1231
|
var pollerFactories = /* @__PURE__ */ new Map();
|
|
1109
1232
|
function registerPollerFactory(type, factory) {
|
|
1110
1233
|
pollerFactories.set(type, factory);
|
|
1111
1234
|
}
|
|
1112
|
-
function createPoller(config, gateway) {
|
|
1113
|
-
const factory = pollerFactories.get(config.type);
|
|
1235
|
+
function createPoller(config, gateway, registry) {
|
|
1236
|
+
const factory = registry ? registry.get(config.type) : pollerFactories.get(config.type);
|
|
1114
1237
|
if (!factory) {
|
|
1115
1238
|
throw new Error(
|
|
1116
1239
|
`No poller factory registered for connector type "${config.type}". Did you import the connector package (e.g. "@lakesync/connector-${config.type}")?`
|
|
@@ -1209,6 +1332,7 @@ async function generateDeltaId(params) {
|
|
|
1209
1332
|
}
|
|
1210
1333
|
|
|
1211
1334
|
// ../core/src/delta/types.ts
|
|
1335
|
+
var COLUMN_TYPES = ["string", "number", "boolean", "json", "null"];
|
|
1212
1336
|
function rowKey(table, rowId) {
|
|
1213
1337
|
return `${table}:${rowId}`;
|
|
1214
1338
|
}
|
|
@@ -1475,6 +1599,9 @@ export {
|
|
|
1475
1599
|
AuthError,
|
|
1476
1600
|
verifyToken,
|
|
1477
1601
|
HLC,
|
|
1602
|
+
ChunkedPusher,
|
|
1603
|
+
PressureManager,
|
|
1604
|
+
PollingScheduler,
|
|
1478
1605
|
isIngestTarget,
|
|
1479
1606
|
BaseSourcePoller,
|
|
1480
1607
|
CallbackPushTarget,
|
|
@@ -1482,16 +1609,19 @@ export {
|
|
|
1482
1609
|
resolveLWW,
|
|
1483
1610
|
isActionHandler,
|
|
1484
1611
|
ConnectorValidationError,
|
|
1612
|
+
createConnectorRegistry,
|
|
1485
1613
|
registerConnectorDescriptor,
|
|
1486
1614
|
registerOutputSchemas,
|
|
1487
1615
|
getConnectorDescriptor,
|
|
1488
1616
|
listConnectorDescriptors,
|
|
1489
1617
|
CONNECTOR_TYPES,
|
|
1490
1618
|
validateConnectorConfig,
|
|
1619
|
+
createPollerRegistry,
|
|
1491
1620
|
registerPollerFactory,
|
|
1492
1621
|
createPoller,
|
|
1493
1622
|
applyDelta,
|
|
1494
1623
|
extractDelta,
|
|
1624
|
+
COLUMN_TYPES,
|
|
1495
1625
|
rowKey,
|
|
1496
1626
|
bigintReplacer,
|
|
1497
1627
|
bigintReviver,
|
|
@@ -1507,4 +1637,4 @@ export {
|
|
|
1507
1637
|
assertValidIdentifier,
|
|
1508
1638
|
quoteIdentifier
|
|
1509
1639
|
};
|
|
1510
|
-
//# sourceMappingURL=chunk-
|
|
1640
|
+
//# sourceMappingURL=chunk-LDFFCG2K.js.map
|