lakesync 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/adapter.d.ts +185 -20
  2. package/dist/adapter.js +13 -3
  3. package/dist/analyst.js +2 -2
  4. package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Bj9kX9dv.d.ts} +76 -19
  5. package/dist/catalogue.d.ts +1 -1
  6. package/dist/catalogue.js +3 -3
  7. package/dist/chunk-DGUM43GV.js +11 -0
  8. package/dist/{chunk-IRJ4QRWV.js → chunk-JI4C4R5H.js} +249 -140
  9. package/dist/chunk-JI4C4R5H.js.map +1 -0
  10. package/dist/{chunk-FHVTUKXL.js → chunk-KVSWLIJR.js} +2 -2
  11. package/dist/{chunk-P3FT7QCW.js → chunk-LDFFCG2K.js} +377 -247
  12. package/dist/chunk-LDFFCG2K.js.map +1 -0
  13. package/dist/{chunk-GUJWMK5P.js → chunk-LPWXOYNS.js} +373 -350
  14. package/dist/chunk-LPWXOYNS.js.map +1 -0
  15. package/dist/{chunk-QMS7TGFL.js → chunk-PYRS74YP.js} +15 -4
  16. package/dist/{chunk-QMS7TGFL.js.map → chunk-PYRS74YP.js.map} +1 -1
  17. package/dist/{chunk-NCZYFZ3B.js → chunk-QNITY4F6.js} +30 -7
  18. package/dist/{chunk-NCZYFZ3B.js.map → chunk-QNITY4F6.js.map} +1 -1
  19. package/dist/{chunk-SF7Y6ZUA.js → chunk-SSICS5KI.js} +2 -2
  20. package/dist/{chunk-UAUQGP3B.js → chunk-TMLG32QV.js} +2 -2
  21. package/dist/client.d.ts +164 -13
  22. package/dist/client.js +310 -163
  23. package/dist/client.js.map +1 -1
  24. package/dist/compactor.d.ts +1 -1
  25. package/dist/compactor.js +4 -4
  26. package/dist/connector-jira.d.ts +2 -2
  27. package/dist/connector-jira.js +3 -3
  28. package/dist/connector-salesforce.d.ts +2 -2
  29. package/dist/connector-salesforce.js +3 -3
  30. package/dist/{coordinator-D32a5rNk.d.ts → coordinator-NXy6tA0h.d.ts} +23 -16
  31. package/dist/{db-types-BlN-4KbQ.d.ts → db-types-CfLMUBfW.d.ts} +1 -1
  32. package/dist/gateway-server.d.ts +158 -64
  33. package/dist/gateway-server.js +482 -4003
  34. package/dist/gateway-server.js.map +1 -1
  35. package/dist/gateway.d.ts +61 -104
  36. package/dist/gateway.js +12 -6
  37. package/dist/index.d.ts +45 -10
  38. package/dist/index.js +14 -2
  39. package/dist/parquet.d.ts +1 -1
  40. package/dist/parquet.js +3 -3
  41. package/dist/proto.d.ts +1 -1
  42. package/dist/proto.js +3 -3
  43. package/dist/react.d.ts +47 -10
  44. package/dist/react.js +88 -40
  45. package/dist/react.js.map +1 -1
  46. package/dist/{registry-CPTgO9jv.d.ts → registry-BcspAtZI.d.ts} +19 -4
  47. package/dist/{gateway-Bpvatd9n.d.ts → request-handler-pUvL7ozF.d.ts} +139 -10
  48. package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
  49. package/dist/{src-FPJQYQNA.js → src-B6NLV3FP.js} +4 -4
  50. package/dist/{src-RHKJFQKR.js → src-ROW4XLO7.js} +15 -3
  51. package/dist/{src-CLCALYDT.js → src-ZRHKG42A.js} +4 -4
  52. package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
  53. package/dist/{types-D-E0VrfS.d.ts → types-BrcD1oJg.d.ts} +26 -19
  54. package/package.json +1 -1
  55. package/dist/chunk-7D4SUZUM.js +0 -38
  56. package/dist/chunk-GUJWMK5P.js.map +0 -1
  57. package/dist/chunk-IRJ4QRWV.js.map +0 -1
  58. package/dist/chunk-P3FT7QCW.js.map +0 -1
  59. /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
  60. /package/dist/{chunk-FHVTUKXL.js.map → chunk-KVSWLIJR.js.map} +0 -0
  61. /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-SSICS5KI.js.map} +0 -0
  62. /package/dist/{chunk-UAUQGP3B.js.map → chunk-TMLG32QV.js.map} +0 -0
  63. /package/dist/{src-CLCALYDT.js.map → src-B6NLV3FP.js.map} +0 -0
  64. /package/dist/{src-FPJQYQNA.js.map → src-ROW4XLO7.js.map} +0 -0
  65. /package/dist/{src-RHKJFQKR.js.map → src-ZRHKG42A.js.map} +0 -0
@@ -387,88 +387,45 @@ var HLC = class _HLC {
387
387
  }
388
388
  };
389
389
 
390
- // ../core/src/base-poller.ts
391
- function isIngestTarget(target) {
392
- return typeof target.flush === "function" && typeof target.shouldFlush === "function" && "bufferStats" in target;
393
- }
394
- var DEFAULT_CHUNK_SIZE = 500;
395
- var DEFAULT_FLUSH_THRESHOLD = 0.7;
396
- var BaseSourcePoller = class {
397
- gateway;
398
- hlc;
390
+ // ../core/src/polling/chunked-pusher.ts
391
+ var ChunkedPusher = class {
392
+ target;
399
393
  clientId;
400
- intervalMs;
401
- timer = null;
402
- running = false;
403
394
  chunkSize;
404
- memoryBudgetBytes;
405
- flushThreshold;
395
+ pressure;
406
396
  pendingDeltas = [];
407
397
  constructor(config) {
408
- this.gateway = config.gateway;
409
- this.hlc = new HLC();
410
- this.clientId = `ingest:${config.name}`;
411
- this.intervalMs = config.intervalMs;
412
- this.chunkSize = config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE;
413
- this.memoryBudgetBytes = config.memory?.memoryBudgetBytes;
414
- this.flushThreshold = config.memory?.flushThreshold ?? DEFAULT_FLUSH_THRESHOLD;
415
- }
416
- /** Start the polling loop. */
417
- start() {
418
- if (this.running) return;
419
- this.running = true;
420
- this.schedulePoll();
421
- }
422
- /** Stop the polling loop. */
423
- stop() {
424
- this.running = false;
425
- if (this.timer) {
426
- clearTimeout(this.timer);
427
- this.timer = null;
428
- }
429
- }
430
- /** Whether the poller is currently running. */
431
- get isRunning() {
432
- return this.running;
433
- }
434
- /**
435
- * Execute a single poll cycle without the timer loop.
436
- * Convenience for serverless consumers who trigger polls manually.
437
- */
438
- async pollOnce() {
439
- return this.poll();
440
- }
441
- /** Push collected deltas to the gateway (single-shot, backward compat). */
442
- pushDeltas(deltas) {
443
- if (deltas.length === 0) return;
444
- const push = {
445
- clientId: this.clientId,
446
- deltas,
447
- lastSeenHlc: 0n
448
- };
449
- this.gateway.handlePush(push);
398
+ this.target = config.target;
399
+ this.clientId = config.clientId;
400
+ this.chunkSize = config.chunkSize;
401
+ this.pressure = config.pressure;
450
402
  }
451
403
  /**
452
404
  * Accumulate a single delta. When `chunkSize` is reached, the pending
453
405
  * deltas are automatically pushed (and flushed if needed).
454
406
  */
455
- async accumulateDelta(delta) {
407
+ async accumulate(delta) {
456
408
  this.pendingDeltas.push(delta);
457
409
  if (this.pendingDeltas.length >= this.chunkSize) {
458
410
  await this.pushPendingChunk();
459
411
  }
460
412
  }
461
- /** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
462
- async flushAccumulator() {
413
+ /** Flush any remaining accumulated deltas. */
414
+ async flush() {
463
415
  if (this.pendingDeltas.length > 0) {
464
416
  await this.pushPendingChunk();
465
417
  }
466
418
  }
467
- /**
468
- * Push a chunk of pending deltas. If the gateway is an IngestTarget,
469
- * checks memory pressure and flushes before/after push when needed.
470
- * On backpressure, flushes once and retries.
471
- */
419
+ /** Push deltas directly (single-shot, backward compat). */
420
+ pushImmediate(deltas) {
421
+ if (deltas.length === 0) return;
422
+ const push = {
423
+ clientId: this.clientId,
424
+ deltas,
425
+ lastSeenHlc: 0n
426
+ };
427
+ this.target.handlePush(push);
428
+ }
472
429
  async pushPendingChunk() {
473
430
  const chunk = this.pendingDeltas;
474
431
  this.pendingDeltas = [];
@@ -476,45 +433,162 @@ var BaseSourcePoller = class {
476
433
  }
477
434
  async pushChunkWithFlush(chunk) {
478
435
  if (chunk.length === 0) return;
479
- const target = this.gateway;
480
- if (isIngestTarget(target)) {
481
- if (this.shouldFlushTarget(target)) {
482
- await target.flush();
483
- }
436
+ if (this.pressure) {
437
+ await this.pressure.checkAndFlush();
484
438
  }
485
439
  const push = {
486
440
  clientId: this.clientId,
487
441
  deltas: chunk,
488
442
  lastSeenHlc: 0n
489
443
  };
490
- const result = target.handlePush(push);
444
+ const result = this.target.handlePush(push);
491
445
  if (result && typeof result === "object" && "ok" in result && !result.ok) {
492
- if (isIngestTarget(target)) {
493
- await target.flush();
494
- target.handlePush(push);
446
+ if (this.pressure) {
447
+ await this.pressure.forceFlush();
448
+ this.target.handlePush(push);
495
449
  }
496
450
  }
497
451
  }
498
- shouldFlushTarget(target) {
499
- if (target.shouldFlush()) return true;
452
+ };
453
+
454
+ // ../core/src/polling/pressure-manager.ts
455
+ var PressureManager = class {
456
+ target;
457
+ memoryBudgetBytes;
458
+ flushThreshold;
459
+ constructor(config) {
460
+ this.target = config.target;
461
+ this.memoryBudgetBytes = config.memoryBudgetBytes;
462
+ this.flushThreshold = config.flushThreshold ?? 0.7;
463
+ }
464
+ /** Check buffer pressure and flush if thresholds are exceeded. */
465
+ async checkAndFlush() {
466
+ if (this.shouldFlush()) {
467
+ await this.target.flush();
468
+ }
469
+ }
470
+ /** Force a flush regardless of current pressure. */
471
+ async forceFlush() {
472
+ await this.target.flush();
473
+ }
474
+ shouldFlush() {
475
+ if (this.target.shouldFlush()) return true;
500
476
  if (this.memoryBudgetBytes != null) {
501
477
  const threshold = Math.floor(this.memoryBudgetBytes * this.flushThreshold);
502
- if (target.bufferStats.byteSize >= threshold) return true;
478
+ if (this.target.bufferStats.byteSize >= threshold) return true;
503
479
  }
504
480
  return false;
505
481
  }
506
- schedulePoll() {
482
+ };
483
+
484
+ // ../core/src/polling/scheduler.ts
485
+ var PollingScheduler = class {
486
+ pollFn;
487
+ intervalMs;
488
+ timer = null;
489
+ running = false;
490
+ constructor(pollFn, intervalMs) {
491
+ this.pollFn = pollFn;
492
+ this.intervalMs = intervalMs;
493
+ }
494
+ /** Start the polling loop. No-op if already running. */
495
+ start() {
496
+ if (this.running) return;
497
+ this.running = true;
498
+ this.schedule();
499
+ }
500
+ /** Stop the polling loop. */
501
+ stop() {
502
+ this.running = false;
503
+ if (this.timer) {
504
+ clearTimeout(this.timer);
505
+ this.timer = null;
506
+ }
507
+ }
508
+ /** Whether the scheduler is currently running. */
509
+ get isRunning() {
510
+ return this.running;
511
+ }
512
+ /** Execute a single poll cycle without the timer loop. */
513
+ async pollOnce() {
514
+ return this.pollFn();
515
+ }
516
+ schedule() {
507
517
  if (!this.running) return;
508
518
  this.timer = setTimeout(async () => {
509
519
  try {
510
- await this.poll();
520
+ await this.pollFn();
511
521
  } catch {
512
522
  }
513
- this.schedulePoll();
523
+ this.schedule();
514
524
  }, this.intervalMs);
515
525
  }
516
526
  };
517
527
 
528
+ // ../core/src/base-poller.ts
529
+ function isIngestTarget(target) {
530
+ return typeof target.flush === "function" && typeof target.shouldFlush === "function" && "bufferStats" in target;
531
+ }
532
+ var DEFAULT_CHUNK_SIZE = 500;
533
+ var BaseSourcePoller = class {
534
+ gateway;
535
+ hlc;
536
+ clientId;
537
+ scheduler;
538
+ pusher;
539
+ constructor(config) {
540
+ this.gateway = config.gateway;
541
+ this.hlc = new HLC();
542
+ this.clientId = `ingest:${config.name}`;
543
+ const pressure = isIngestTarget(config.gateway) ? new PressureManager({
544
+ target: config.gateway,
545
+ memoryBudgetBytes: config.memory?.memoryBudgetBytes,
546
+ flushThreshold: config.memory?.flushThreshold
547
+ }) : null;
548
+ this.pusher = new ChunkedPusher({
549
+ target: config.gateway,
550
+ clientId: this.clientId,
551
+ chunkSize: config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE,
552
+ pressure
553
+ });
554
+ this.scheduler = new PollingScheduler(() => this.poll(), config.intervalMs);
555
+ }
556
+ /** Start the polling loop. */
557
+ start() {
558
+ this.scheduler.start();
559
+ }
560
+ /** Stop the polling loop. */
561
+ stop() {
562
+ this.scheduler.stop();
563
+ }
564
+ /** Whether the poller is currently running. */
565
+ get isRunning() {
566
+ return this.scheduler.isRunning;
567
+ }
568
+ /**
569
+ * Execute a single poll cycle without the timer loop.
570
+ * Convenience for serverless consumers who trigger polls manually.
571
+ */
572
+ async pollOnce() {
573
+ return this.scheduler.pollOnce();
574
+ }
575
+ /** Push collected deltas to the gateway (single-shot, backward compat). */
576
+ pushDeltas(deltas) {
577
+ this.pusher.pushImmediate(deltas);
578
+ }
579
+ /**
580
+ * Accumulate a single delta. When `chunkSize` is reached, the pending
581
+ * deltas are automatically pushed (and flushed if needed).
582
+ */
583
+ async accumulateDelta(delta) {
584
+ await this.pusher.accumulate(delta);
585
+ }
586
+ /** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
587
+ async flushAccumulator() {
588
+ await this.pusher.flush();
589
+ }
590
+ };
591
+
518
592
  // ../core/src/callback-push-target.ts
519
593
  var CallbackPushTarget = class {
520
594
  onPush;
@@ -621,6 +695,35 @@ var ConnectorValidationError = class extends LakeSyncError {
621
695
  };
622
696
 
623
697
  // ../core/src/connector/registry.ts
698
+ function createConnectorRegistry(descriptors2) {
699
+ const map = /* @__PURE__ */ new Map();
700
+ for (const d of descriptors2) {
701
+ map.set(d.type, d);
702
+ }
703
+ return buildRegistry(map);
704
+ }
705
+ function buildRegistry(map) {
706
+ return {
707
+ get(type) {
708
+ return map.get(type);
709
+ },
710
+ list() {
711
+ return [...map.values()].sort((a, b) => a.type.localeCompare(b.type));
712
+ },
713
+ with(descriptor) {
714
+ const next = new Map(map);
715
+ next.set(descriptor.type, descriptor);
716
+ return buildRegistry(next);
717
+ },
718
+ withOutputSchemas(type, schemas) {
719
+ const existing = map.get(type);
720
+ if (!existing) return buildRegistry(map);
721
+ const next = new Map(map);
722
+ next.set(type, { ...existing, outputTables: schemas });
723
+ return buildRegistry(next);
724
+ }
725
+ };
726
+ }
624
727
  var descriptors = /* @__PURE__ */ new Map();
625
728
  function registerConnectorDescriptor(descriptor) {
626
729
  descriptors.set(descriptor.type, descriptor);
@@ -642,6 +745,162 @@ var CONNECTOR_TYPES = ["postgres", "mysql", "bigquery", "jira", "salesforce"];
642
745
 
643
746
  // ../core/src/connector/validate.ts
644
747
  var VALID_STRATEGIES = /* @__PURE__ */ new Set(["cursor", "diff"]);
748
+ function validatePostgresConfig(obj) {
749
+ const pg = obj.postgres;
750
+ if (typeof pg !== "object" || pg === null) {
751
+ return Err(
752
+ new ConnectorValidationError('Connector type "postgres" requires a postgres config object')
753
+ );
754
+ }
755
+ const pgObj = pg;
756
+ if (typeof pgObj.connectionString !== "string" || pgObj.connectionString.length === 0) {
757
+ return Err(
758
+ new ConnectorValidationError("Postgres connector requires a non-empty connectionString")
759
+ );
760
+ }
761
+ return Ok(void 0);
762
+ }
763
+ function validateMySQLConfig(obj) {
764
+ const my = obj.mysql;
765
+ if (typeof my !== "object" || my === null) {
766
+ return Err(
767
+ new ConnectorValidationError('Connector type "mysql" requires a mysql config object')
768
+ );
769
+ }
770
+ const myObj = my;
771
+ if (typeof myObj.connectionString !== "string" || myObj.connectionString.length === 0) {
772
+ return Err(
773
+ new ConnectorValidationError("MySQL connector requires a non-empty connectionString")
774
+ );
775
+ }
776
+ return Ok(void 0);
777
+ }
778
+ function validateBigQueryConfig(obj) {
779
+ const bq = obj.bigquery;
780
+ if (typeof bq !== "object" || bq === null) {
781
+ return Err(
782
+ new ConnectorValidationError('Connector type "bigquery" requires a bigquery config object')
783
+ );
784
+ }
785
+ const bqObj = bq;
786
+ if (typeof bqObj.projectId !== "string" || bqObj.projectId.length === 0) {
787
+ return Err(new ConnectorValidationError("BigQuery connector requires a non-empty projectId"));
788
+ }
789
+ if (typeof bqObj.dataset !== "string" || bqObj.dataset.length === 0) {
790
+ return Err(new ConnectorValidationError("BigQuery connector requires a non-empty dataset"));
791
+ }
792
+ return Ok(void 0);
793
+ }
794
+ function validateJiraConfig(obj) {
795
+ const jira = obj.jira;
796
+ if (typeof jira !== "object" || jira === null) {
797
+ return Err(new ConnectorValidationError('Connector type "jira" requires a jira config object'));
798
+ }
799
+ const jiraObj = jira;
800
+ if (typeof jiraObj.domain !== "string" || jiraObj.domain.length === 0) {
801
+ return Err(new ConnectorValidationError("Jira connector requires a non-empty domain"));
802
+ }
803
+ if (typeof jiraObj.email !== "string" || jiraObj.email.length === 0) {
804
+ return Err(new ConnectorValidationError("Jira connector requires a non-empty email"));
805
+ }
806
+ if (typeof jiraObj.apiToken !== "string" || jiraObj.apiToken.length === 0) {
807
+ return Err(new ConnectorValidationError("Jira connector requires a non-empty apiToken"));
808
+ }
809
+ return Ok(void 0);
810
+ }
811
+ function validateSalesforceConfig(obj) {
812
+ const sf = obj.salesforce;
813
+ if (typeof sf !== "object" || sf === null) {
814
+ return Err(
815
+ new ConnectorValidationError(
816
+ 'Connector type "salesforce" requires a salesforce config object'
817
+ )
818
+ );
819
+ }
820
+ const sfObj = sf;
821
+ if (typeof sfObj.instanceUrl !== "string" || sfObj.instanceUrl.length === 0) {
822
+ return Err(
823
+ new ConnectorValidationError("Salesforce connector requires a non-empty instanceUrl")
824
+ );
825
+ }
826
+ if (typeof sfObj.clientId !== "string" || sfObj.clientId.length === 0) {
827
+ return Err(new ConnectorValidationError("Salesforce connector requires a non-empty clientId"));
828
+ }
829
+ if (typeof sfObj.clientSecret !== "string" || sfObj.clientSecret.length === 0) {
830
+ return Err(
831
+ new ConnectorValidationError("Salesforce connector requires a non-empty clientSecret")
832
+ );
833
+ }
834
+ if (typeof sfObj.username !== "string" || sfObj.username.length === 0) {
835
+ return Err(new ConnectorValidationError("Salesforce connector requires a non-empty username"));
836
+ }
837
+ if (typeof sfObj.password !== "string" || sfObj.password.length === 0) {
838
+ return Err(new ConnectorValidationError("Salesforce connector requires a non-empty password"));
839
+ }
840
+ return Ok(void 0);
841
+ }
842
+ function validateIngestConfig(obj, connectorType) {
843
+ if (obj.ingest === void 0) return Ok(void 0);
844
+ if (typeof obj.ingest !== "object" || obj.ingest === null) {
845
+ return Err(new ConnectorValidationError("Ingest config must be an object"));
846
+ }
847
+ const ingest = obj.ingest;
848
+ if (connectorType === "jira" || connectorType === "salesforce") {
849
+ if (ingest.intervalMs !== void 0) {
850
+ if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
851
+ return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
852
+ }
853
+ }
854
+ return Ok(void 0);
855
+ }
856
+ if (!Array.isArray(ingest.tables) || ingest.tables.length === 0) {
857
+ return Err(new ConnectorValidationError("Ingest config must have a non-empty tables array"));
858
+ }
859
+ for (let i = 0; i < ingest.tables.length; i++) {
860
+ const table = ingest.tables[i];
861
+ if (typeof table !== "object" || table === null) {
862
+ return Err(new ConnectorValidationError(`Ingest table at index ${i} must be an object`));
863
+ }
864
+ if (typeof table.table !== "string" || table.table.length === 0) {
865
+ return Err(
866
+ new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty table name`)
867
+ );
868
+ }
869
+ if (typeof table.query !== "string" || table.query.length === 0) {
870
+ return Err(
871
+ new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty query`)
872
+ );
873
+ }
874
+ if (typeof table.strategy !== "object" || table.strategy === null) {
875
+ return Err(
876
+ new ConnectorValidationError(`Ingest table at index ${i} must have a strategy object`)
877
+ );
878
+ }
879
+ const strategy = table.strategy;
880
+ if (!VALID_STRATEGIES.has(strategy.type)) {
881
+ return Err(
882
+ new ConnectorValidationError(
883
+ `Ingest table at index ${i} strategy type must be "cursor" or "diff"`
884
+ )
885
+ );
886
+ }
887
+ if (strategy.type === "cursor") {
888
+ if (typeof strategy.cursorColumn !== "string" || strategy.cursorColumn.length === 0) {
889
+ return Err(
890
+ new ConnectorValidationError(
891
+ `Ingest table at index ${i} cursor strategy requires a non-empty cursorColumn`
892
+ )
893
+ );
894
+ }
895
+ }
896
+ }
897
+ if (ingest.intervalMs !== void 0) {
898
+ if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
899
+ return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
900
+ }
901
+ }
902
+ return Ok(void 0);
903
+ }
645
904
  function validateConnectorConfig(input) {
646
905
  if (typeof input !== "object" || input === null) {
647
906
  return Err(new ConnectorValidationError("Connector config must be an object"));
@@ -656,178 +915,27 @@ function validateConnectorConfig(input) {
656
915
  );
657
916
  }
658
917
  const connectorType = obj.type;
918
+ let typeResult;
659
919
  switch (connectorType) {
660
- case "postgres": {
661
- const pg = obj.postgres;
662
- if (typeof pg !== "object" || pg === null) {
663
- return Err(
664
- new ConnectorValidationError(
665
- 'Connector type "postgres" requires a postgres config object'
666
- )
667
- );
668
- }
669
- const pgObj = pg;
670
- if (typeof pgObj.connectionString !== "string" || pgObj.connectionString.length === 0) {
671
- return Err(
672
- new ConnectorValidationError("Postgres connector requires a non-empty connectionString")
673
- );
674
- }
920
+ case "postgres":
921
+ typeResult = validatePostgresConfig(obj);
675
922
  break;
676
- }
677
- case "mysql": {
678
- const my = obj.mysql;
679
- if (typeof my !== "object" || my === null) {
680
- return Err(
681
- new ConnectorValidationError('Connector type "mysql" requires a mysql config object')
682
- );
683
- }
684
- const myObj = my;
685
- if (typeof myObj.connectionString !== "string" || myObj.connectionString.length === 0) {
686
- return Err(
687
- new ConnectorValidationError("MySQL connector requires a non-empty connectionString")
688
- );
689
- }
923
+ case "mysql":
924
+ typeResult = validateMySQLConfig(obj);
690
925
  break;
691
- }
692
- case "bigquery": {
693
- const bq = obj.bigquery;
694
- if (typeof bq !== "object" || bq === null) {
695
- return Err(
696
- new ConnectorValidationError(
697
- 'Connector type "bigquery" requires a bigquery config object'
698
- )
699
- );
700
- }
701
- const bqObj = bq;
702
- if (typeof bqObj.projectId !== "string" || bqObj.projectId.length === 0) {
703
- return Err(
704
- new ConnectorValidationError("BigQuery connector requires a non-empty projectId")
705
- );
706
- }
707
- if (typeof bqObj.dataset !== "string" || bqObj.dataset.length === 0) {
708
- return Err(new ConnectorValidationError("BigQuery connector requires a non-empty dataset"));
709
- }
926
+ case "bigquery":
927
+ typeResult = validateBigQueryConfig(obj);
710
928
  break;
711
- }
712
- case "jira": {
713
- const jira = obj.jira;
714
- if (typeof jira !== "object" || jira === null) {
715
- return Err(
716
- new ConnectorValidationError('Connector type "jira" requires a jira config object')
717
- );
718
- }
719
- const jiraObj = jira;
720
- if (typeof jiraObj.domain !== "string" || jiraObj.domain.length === 0) {
721
- return Err(new ConnectorValidationError("Jira connector requires a non-empty domain"));
722
- }
723
- if (typeof jiraObj.email !== "string" || jiraObj.email.length === 0) {
724
- return Err(new ConnectorValidationError("Jira connector requires a non-empty email"));
725
- }
726
- if (typeof jiraObj.apiToken !== "string" || jiraObj.apiToken.length === 0) {
727
- return Err(new ConnectorValidationError("Jira connector requires a non-empty apiToken"));
728
- }
929
+ case "jira":
930
+ typeResult = validateJiraConfig(obj);
729
931
  break;
730
- }
731
- case "salesforce": {
732
- const sf = obj.salesforce;
733
- if (typeof sf !== "object" || sf === null) {
734
- return Err(
735
- new ConnectorValidationError(
736
- 'Connector type "salesforce" requires a salesforce config object'
737
- )
738
- );
739
- }
740
- const sfObj = sf;
741
- if (typeof sfObj.instanceUrl !== "string" || sfObj.instanceUrl.length === 0) {
742
- return Err(
743
- new ConnectorValidationError("Salesforce connector requires a non-empty instanceUrl")
744
- );
745
- }
746
- if (typeof sfObj.clientId !== "string" || sfObj.clientId.length === 0) {
747
- return Err(
748
- new ConnectorValidationError("Salesforce connector requires a non-empty clientId")
749
- );
750
- }
751
- if (typeof sfObj.clientSecret !== "string" || sfObj.clientSecret.length === 0) {
752
- return Err(
753
- new ConnectorValidationError("Salesforce connector requires a non-empty clientSecret")
754
- );
755
- }
756
- if (typeof sfObj.username !== "string" || sfObj.username.length === 0) {
757
- return Err(
758
- new ConnectorValidationError("Salesforce connector requires a non-empty username")
759
- );
760
- }
761
- if (typeof sfObj.password !== "string" || sfObj.password.length === 0) {
762
- return Err(
763
- new ConnectorValidationError("Salesforce connector requires a non-empty password")
764
- );
765
- }
932
+ case "salesforce":
933
+ typeResult = validateSalesforceConfig(obj);
766
934
  break;
767
- }
768
- }
769
- if (obj.ingest !== void 0) {
770
- if (typeof obj.ingest !== "object" || obj.ingest === null) {
771
- return Err(new ConnectorValidationError("Ingest config must be an object"));
772
- }
773
- const ingest = obj.ingest;
774
- if (connectorType === "jira" || connectorType === "salesforce") {
775
- if (ingest.intervalMs !== void 0) {
776
- if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
777
- return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
778
- }
779
- }
780
- return Ok(input);
781
- }
782
- if (!Array.isArray(ingest.tables) || ingest.tables.length === 0) {
783
- return Err(new ConnectorValidationError("Ingest config must have a non-empty tables array"));
784
- }
785
- for (let i = 0; i < ingest.tables.length; i++) {
786
- const table = ingest.tables[i];
787
- if (typeof table !== "object" || table === null) {
788
- return Err(new ConnectorValidationError(`Ingest table at index ${i} must be an object`));
789
- }
790
- if (typeof table.table !== "string" || table.table.length === 0) {
791
- return Err(
792
- new ConnectorValidationError(
793
- `Ingest table at index ${i} must have a non-empty table name`
794
- )
795
- );
796
- }
797
- if (typeof table.query !== "string" || table.query.length === 0) {
798
- return Err(
799
- new ConnectorValidationError(`Ingest table at index ${i} must have a non-empty query`)
800
- );
801
- }
802
- if (typeof table.strategy !== "object" || table.strategy === null) {
803
- return Err(
804
- new ConnectorValidationError(`Ingest table at index ${i} must have a strategy object`)
805
- );
806
- }
807
- const strategy = table.strategy;
808
- if (!VALID_STRATEGIES.has(strategy.type)) {
809
- return Err(
810
- new ConnectorValidationError(
811
- `Ingest table at index ${i} strategy type must be "cursor" or "diff"`
812
- )
813
- );
814
- }
815
- if (strategy.type === "cursor") {
816
- if (typeof strategy.cursorColumn !== "string" || strategy.cursorColumn.length === 0) {
817
- return Err(
818
- new ConnectorValidationError(
819
- `Ingest table at index ${i} cursor strategy requires a non-empty cursorColumn`
820
- )
821
- );
822
- }
823
- }
824
- }
825
- if (ingest.intervalMs !== void 0) {
826
- if (typeof ingest.intervalMs !== "number" || ingest.intervalMs < 1) {
827
- return Err(new ConnectorValidationError("Ingest intervalMs must be a positive number"));
828
- }
829
- }
830
935
  }
936
+ if (!typeResult.ok) return typeResult;
937
+ const ingestResult = validateIngestConfig(obj, connectorType);
938
+ if (!ingestResult.ok) return ingestResult;
831
939
  return Ok(input);
832
940
  }
833
941
 
@@ -1105,12 +1213,27 @@ registerConnectorDescriptor({
1105
1213
  });
1106
1214
 
1107
1215
  // ../core/src/create-poller.ts
1216
+ function createPollerRegistry(factories = /* @__PURE__ */ new Map()) {
1217
+ return buildPollerRegistry(new Map(factories));
1218
+ }
1219
+ function buildPollerRegistry(map) {
1220
+ return {
1221
+ get(type) {
1222
+ return map.get(type);
1223
+ },
1224
+ with(type, factory) {
1225
+ const next = new Map(map);
1226
+ next.set(type, factory);
1227
+ return buildPollerRegistry(next);
1228
+ }
1229
+ };
1230
+ }
1108
1231
  var pollerFactories = /* @__PURE__ */ new Map();
1109
1232
  function registerPollerFactory(type, factory) {
1110
1233
  pollerFactories.set(type, factory);
1111
1234
  }
1112
- function createPoller(config, gateway) {
1113
- const factory = pollerFactories.get(config.type);
1235
+ function createPoller(config, gateway, registry) {
1236
+ const factory = registry ? registry.get(config.type) : pollerFactories.get(config.type);
1114
1237
  if (!factory) {
1115
1238
  throw new Error(
1116
1239
  `No poller factory registered for connector type "${config.type}". Did you import the connector package (e.g. "@lakesync/connector-${config.type}")?`
@@ -1209,6 +1332,7 @@ async function generateDeltaId(params) {
1209
1332
  }
1210
1333
 
1211
1334
  // ../core/src/delta/types.ts
1335
+ var COLUMN_TYPES = ["string", "number", "boolean", "json", "null"];
1212
1336
  function rowKey(table, rowId) {
1213
1337
  return `${table}:${rowId}`;
1214
1338
  }
@@ -1475,6 +1599,9 @@ export {
1475
1599
  AuthError,
1476
1600
  verifyToken,
1477
1601
  HLC,
1602
+ ChunkedPusher,
1603
+ PressureManager,
1604
+ PollingScheduler,
1478
1605
  isIngestTarget,
1479
1606
  BaseSourcePoller,
1480
1607
  CallbackPushTarget,
@@ -1482,16 +1609,19 @@ export {
1482
1609
  resolveLWW,
1483
1610
  isActionHandler,
1484
1611
  ConnectorValidationError,
1612
+ createConnectorRegistry,
1485
1613
  registerConnectorDescriptor,
1486
1614
  registerOutputSchemas,
1487
1615
  getConnectorDescriptor,
1488
1616
  listConnectorDescriptors,
1489
1617
  CONNECTOR_TYPES,
1490
1618
  validateConnectorConfig,
1619
+ createPollerRegistry,
1491
1620
  registerPollerFactory,
1492
1621
  createPoller,
1493
1622
  applyDelta,
1494
1623
  extractDelta,
1624
+ COLUMN_TYPES,
1495
1625
  rowKey,
1496
1626
  bigintReplacer,
1497
1627
  bigintReviver,
@@ -1507,4 +1637,4 @@ export {
1507
1637
  assertValidIdentifier,
1508
1638
  quoteIdentifier
1509
1639
  };
1510
- //# sourceMappingURL=chunk-P3FT7QCW.js.map
1640
+ //# sourceMappingURL=chunk-LDFFCG2K.js.map