@whaly/connector-sdk 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -38,6 +38,7 @@ __export(index_exports, {
38
38
  CounterMetric: () => CounterMetric,
39
39
  DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
40
40
  EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
41
+ GCSStateProvider: () => GCSStateProvider,
41
42
  ITarget: () => ITarget,
42
43
  MissingFieldInSchemaError: () => MissingFieldInSchemaError,
43
44
  MissingSchemaError: () => MissingSchemaError,
@@ -45,14 +46,13 @@ __export(index_exports, {
45
46
  ROWS_SYNCED_METRIC_NAME: () => ROWS_SYNCED_METRIC_NAME,
46
47
  RecordMessage: () => RecordMessage,
47
48
  RenameColumnStore: () => RenameColumnStore,
49
+ ReplicationMethod: () => ReplicationMethod,
48
50
  ReplicationMethodMessage: () => ReplicationMethodMessage,
49
- Resolver: () => Resolver,
50
51
  SchemaMessage: () => SchemaMessage,
51
52
  SchemaValidationError: () => SchemaValidationError,
52
53
  StateMessage: () => StateMessage,
53
54
  StateService: () => StateService,
54
55
  Stream: () => Stream,
55
- StreamMetadata: () => StreamMetadata,
56
56
  StreamWarehouseSyncService: () => StreamWarehouseSyncService,
57
57
  Tap: () => Tap,
58
58
  _greaterThan: () => _greaterThan,
@@ -60,7 +60,6 @@ __export(index_exports, {
60
60
  createTemporaryFileStream: () => createTemporaryFileStream,
61
61
  extractStateForStream: () => extractStateForStream,
62
62
  finalizeStateProgressMarkers: () => finalizeStateProgressMarkers,
63
- findRelatedRelationships: () => findRelatedRelationships,
64
63
  flattenSchema: () => flattenSchema,
65
64
  getAllMetrics: () => getAllMetrics,
66
65
  getAxiosInstance: () => getAxiosInstance,
@@ -74,7 +73,6 @@ __export(index_exports, {
74
73
  haltAndCatchFire: () => haltAndCatchFire,
75
74
  incrementStreamState: () => incrementStreamState,
76
75
  loadJson: () => loadJson,
77
- loadResolver: () => loadResolver,
78
76
  loadSchema: () => loadSchema,
79
77
  logger: () => logger,
80
78
  postFormDataApiCall: () => postFormDataApiCall,
@@ -82,189 +80,18 @@ __export(index_exports, {
82
80
  postUrlEncodedApiCall: () => postUrlEncodedApiCall,
83
81
  printMemoryFootprint: () => printMemoryFootprint,
84
82
  removeParasiteProperties: () => removeParasiteProperties,
85
- safePath: () => safePath,
86
- writeStateFile: () => writeStateFile
83
+ safePath: () => safePath
87
84
  });
88
85
  module.exports = __toCommonJS(index_exports);
89
86
 
90
87
  // src/sdk/utils.ts
91
- var import_fs2 = require("fs");
92
-
93
- // src/sdk/service/logger.ts
94
- var winston2 = __toESM(require("winston"));
95
-
96
- // src/sdk/models/resolver.ts
97
- var Resolver = class {
98
- constructor() {
99
- }
100
- };
101
-
102
- // src/resolvers/local-file/services/files.ts
103
- var import_fs = __toESM(require("fs"));
104
- var writeFile = (path, content) => {
105
- return Promise.resolve(import_fs.default.writeFileSync(path, content));
106
- };
107
- var readAndParseJSONFile = (path) => {
108
- if (!import_fs.default.existsSync(path)) {
109
- throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
110
- }
111
- const raw = import_fs.default.readFileSync(path);
112
- const parsed = JSON.parse(raw.toString());
113
- return parsed;
114
- };
115
-
116
- // src/resolvers/local-file/hook/schemaHook.ts
117
- var import_fs_extra = __toESM(require("fs-extra"));
118
- var LocalSchemaHook = class {
119
- async writeSchema(input) {
120
- await import_fs_extra.default.appendFile("./schema.tmp", JSON.stringify(input) + "\n");
121
- }
122
- };
123
-
124
- // src/resolvers/local-file/resolver.ts
125
- var import_winston = __toESM(require("winston"));
126
-
127
- // src/sdk/service/metric.ts
128
- var metricStore = {};
129
- var ALL_STREAM_ID_KEYWORD = "all";
130
- var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
131
- var API_CALLS_METRIC_NAME = "api_calls_count";
132
- var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
133
- var getCounterMetrics = (name, streamIds) => {
134
- return streamIds.map((streamId) => {
135
- return getCounterMetric(name, streamId);
136
- });
137
- };
138
- var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
139
- if (!metricStore[streamId]) {
140
- metricStore[streamId] = {};
141
- }
142
- const streamMetricStore = metricStore[streamId];
143
- if (!streamMetricStore[name]) {
144
- streamMetricStore[name] = new CounterMetric(name, streamId);
145
- }
146
- return streamMetricStore[name];
147
- };
148
- var getAllMetrics = () => {
149
- return Object.keys(metricStore).flatMap((streamId) => {
150
- const streamMetricStore = metricStore[streamId];
151
- if (!streamMetricStore) {
152
- return [];
153
- }
154
- return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
155
- });
156
- };
157
- var CounterMetric = class {
158
- value = 0;
159
- name;
160
- streamId;
161
- constructor(name, streamId) {
162
- this.name = name;
163
- this.streamId = streamId;
164
- }
165
- increment(inc = 1) {
166
- this.value = this.value + inc;
167
- }
168
- getStreamId() {
169
- return this.streamId;
170
- }
171
- getName() {
172
- return this.name;
173
- }
174
- getValue() {
175
- return this.value;
176
- }
177
- };
178
-
179
- // src/resolvers/local-file/resolver.ts
180
- var { combine, prettyPrint, timestamp, errors, splat } = import_winston.default.format;
181
- var statePath = "./state.json";
182
- var metricsPath = "./metrics.json";
183
- var LocalFilesResolver = class extends Resolver {
184
- checkIfCanSync() {
185
- return Promise.resolve(true);
186
- }
187
- markSyncStarted() {
188
- return Promise.resolve();
189
- }
190
- onError(errorType, errorText, errorDebugText) {
191
- const path = `./error.json`;
192
- return writeFile(path, JSON.stringify({
193
- errorType,
194
- errorText,
195
- errorDebugText
196
- }));
197
- }
198
- getSchemaHooks() {
199
- const wlyHook = new LocalSchemaHook();
200
- return [wlyHook];
201
- }
202
- writeSchema() {
203
- return Promise.resolve();
204
- }
205
- constructor() {
206
- super();
207
- }
208
- getState() {
209
- logger.info(`\u{1F4C1} Using the LOCAL resolver`);
210
- const state = readAndParseJSONFile(statePath);
211
- return Promise.resolve({
212
- state
213
- });
214
- }
215
- writeCatalog(catalog) {
216
- return Promise.resolve();
217
- }
218
- writeState(state) {
219
- return writeFile(statePath, state);
220
- }
221
- markSyncFailed() {
222
- logger.info(`\u{1F622} Sync has failed.`);
223
- return Promise.resolve();
224
- }
225
- markSyncComplete() {
226
- logger.info(`\u{1F389} Sync is complete.`);
227
- return Promise.resolve();
228
- }
229
- markDiscoveryComplete() {
230
- logger.info(`\u{1F389} Discovery is complete.`);
231
- return Promise.resolve();
232
- }
233
- async flushMetrics() {
234
- const allMetrics = getAllMetrics();
235
- const metricsOuput = allMetrics.map((metric) => {
236
- return {
237
- streamId: metric.getStreamId(),
238
- name: metric.getName(),
239
- value: metric.getValue()
240
- };
241
- }).map((obj) => JSON.stringify(obj)).join(`
242
- `);
243
- return writeFile(metricsPath, metricsOuput);
244
- }
245
- updateSourceValue(optionKey, optionValue) {
246
- logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
247
- return Promise.resolve();
248
- }
249
- getLogFormat() {
250
- return combine(
251
- timestamp(),
252
- errors({ stack: true }),
253
- splat(),
254
- prettyPrint()
255
- );
256
- }
257
- };
258
-
259
- // src/sdk/service/resolver.ts
260
- var loadResolver = () => {
261
- return new LocalFilesResolver();
262
- };
88
+ var import_fs = require("fs");
263
89
 
264
90
  // src/sdk/service/logger.ts
91
+ var winston = __toESM(require("winston"));
265
92
  var BATCH_INTERVAL_MS = 100;
266
93
  var transports2 = {
267
- console: new winston2.transports.Console({
94
+ console: new winston.transports.Console({
268
95
  stderrLevels: [],
269
96
  level: "info"
270
97
  })
@@ -274,8 +101,14 @@ var getTransports = () => {
274
101
  transports2.console
275
102
  ];
276
103
  };
277
- var winstonLogger = winston2.createLogger({
278
- format: loadResolver().getLogFormat(),
104
+ var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
105
+ var winstonLogger = winston.createLogger({
106
+ format: combine(
107
+ timestamp(),
108
+ errors({ stack: true }),
109
+ splat(),
110
+ prettyPrint()
111
+ ),
279
112
  transports: getTransports()
280
113
  });
281
114
  var isWinstonOpen = true;
@@ -304,13 +137,9 @@ var logger = {
304
137
  };
305
138
 
306
139
  // src/sdk/utils.ts
307
- var writeStateFile = (resolver, state) => {
308
- logger.info(`\u{1F4DD} Writing the state file.`);
309
- return resolver.writeState(state);
310
- };
311
140
  function readFile(fileName, filePath) {
312
141
  try {
313
- return (0, import_fs2.readFileSync)(filePath);
142
+ return (0, import_fs.readFileSync)(filePath);
314
143
  } catch (err) {
315
144
  logger.error(`Can't open file: \`${fileName}\` at path: \`${filePath}\`. Is it at the proper place?`);
316
145
  throw err;
@@ -337,7 +166,7 @@ var loadSchema = (streamId) => {
337
166
  // src/sdk/service/network.ts
338
167
  var import_axios = __toESM(require("axios"));
339
168
  var import_url = require("url");
340
- var fs2 = __toESM(require("fs"));
169
+ var fs = __toESM(require("fs"));
341
170
  var qs = __toESM(require("qs"));
342
171
  var import_axios_retry = __toESM(require("axios-retry"));
343
172
  var shouldRetry = (error) => {
@@ -450,7 +279,7 @@ var getJSONApiCall = async (endpoint, config, privateLogging) => {
450
279
  };
451
280
  var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
452
281
  const instance = getAxiosInstance(config.retryCount);
453
- const writer = fs2.createWriteStream(output);
282
+ const writer = fs.createWriteStream(output);
454
283
  const paramsSerializer = (params) => {
455
284
  return qs.stringify(params, { arrayFormat: "repeat" });
456
285
  };
@@ -544,6 +373,58 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
544
373
  }
545
374
  };
546
375
 
376
+ // src/sdk/service/metric.ts
377
+ var metricStore = {};
378
+ var ALL_STREAM_ID_KEYWORD = "all";
379
+ var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
380
+ var API_CALLS_METRIC_NAME = "api_calls_count";
381
+ var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
382
+ var getCounterMetrics = (name, streamIds) => {
383
+ return streamIds.map((streamId) => {
384
+ return getCounterMetric(name, streamId);
385
+ });
386
+ };
387
+ var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
388
+ if (!metricStore[streamId]) {
389
+ metricStore[streamId] = {};
390
+ }
391
+ const streamMetricStore = metricStore[streamId];
392
+ if (!streamMetricStore[name]) {
393
+ streamMetricStore[name] = new CounterMetric(name, streamId);
394
+ }
395
+ return streamMetricStore[name];
396
+ };
397
+ var getAllMetrics = () => {
398
+ return Object.keys(metricStore).flatMap((streamId) => {
399
+ const streamMetricStore = metricStore[streamId];
400
+ if (!streamMetricStore) {
401
+ return [];
402
+ }
403
+ return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
404
+ });
405
+ };
406
+ var CounterMetric = class {
407
+ value = 0;
408
+ name;
409
+ streamId;
410
+ constructor(name, streamId) {
411
+ this.name = name;
412
+ this.streamId = streamId;
413
+ }
414
+ increment(inc = 1) {
415
+ this.value = this.value + inc;
416
+ }
417
+ getStreamId() {
418
+ return this.streamId;
419
+ }
420
+ getName() {
421
+ return this.name;
422
+ }
423
+ getValue() {
424
+ return this.value;
425
+ }
426
+ };
427
+
547
428
  // src/sdk/service/memory.ts
548
429
  var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
549
430
  var printMemoryFootprint = (prefix) => {
@@ -559,9 +440,6 @@ var printMemoryFootprint = (prefix) => {
559
440
 
560
441
  // src/sdk/service/exit.ts
561
442
  async function gracefulExit(exitCode) {
562
- if (exitCode !== 0) {
563
- await loadResolver().markSyncFailed();
564
- }
565
443
  const loggerFinish = new Promise((resolve, reject) => {
566
444
  logger.closeWinston(resolve);
567
445
  });
@@ -581,34 +459,15 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
581
459
  errorText,
582
460
  errorDebugText
583
461
  );
584
- const resolver = loadResolver();
585
- await resolver.onError(errorType, errorText, errorDebugText);
586
462
  await gracefulExit(0);
587
463
  };
588
464
 
589
- // src/sdk/service/relationship.ts
590
- var findRelatedRelationships = (streamId, allRels) => {
591
- return allRels.reduce((acc, rel) => {
592
- const base = {
593
- type: rel.type,
594
- from: rel.from,
595
- to: rel.to
596
- };
597
- if (rel.left === streamId) {
598
- acc.right.push({
599
- streamId: rel.right,
600
- ...base
601
- });
602
- }
603
- if (rel.right === streamId) {
604
- acc.left.push({
605
- streamId: rel.left,
606
- ...base
607
- });
608
- }
609
- return acc;
610
- }, { left: [], right: [] });
611
- };
465
+ // src/sdk/models/replication.ts
466
+ var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
467
+ ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
468
+ ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
469
+ return ReplicationMethod2;
470
+ })(ReplicationMethod || {});
612
471
 
613
472
  // src/sdk/models/messages.ts
614
473
  var RecordMessage = class {
@@ -668,7 +527,6 @@ var SchemaMessage = class {
668
527
  displayLabel;
669
528
  description;
670
529
  propertiesMetadata;
671
- relationships;
672
530
  constructor(opts) {
673
531
  const {
674
532
  stream,
@@ -677,7 +535,6 @@ var SchemaMessage = class {
677
535
  bookmarkProperties,
678
536
  displayLabel,
679
537
  description,
680
- relationships,
681
538
  propertiesMetadata
682
539
  } = opts;
683
540
  this.stream = stream;
@@ -686,7 +543,6 @@ var SchemaMessage = class {
686
543
  this.bookmarkProperties = bookmarkProperties;
687
544
  this.displayLabel = displayLabel;
688
545
  this.description = description;
689
- this.relationships = relationships;
690
546
  this.propertiesMetadata = propertiesMetadata;
691
547
  }
692
548
  toString() {
@@ -696,8 +552,7 @@ var SchemaMessage = class {
696
552
  schema: this.schema,
697
553
  key_properties: this.keyProperties,
698
554
  label: this.displayLabel,
699
- description: this.description,
700
- relationships: this.relationships
555
+ description: this.description
701
556
  };
702
557
  if (this.bookmarkProperties) {
703
558
  result["bookmark_properties"] = this.bookmarkProperties;
@@ -706,128 +561,8 @@ var SchemaMessage = class {
706
561
  }
707
562
  };
708
563
 
709
- // src/sdk/models/metadata.ts
710
- var MetadataMap = class extends Map {
711
- areEquals(array1, array2) {
712
- return array1.length === array2.length && array1.every(function(value, index) {
713
- return value === array2[index];
714
- });
715
- }
716
- findExistingEntry(key) {
717
- return Array.from(super.entries()).find((entry) => {
718
- if (this.areEquals(entry[0], key)) {
719
- return true;
720
- }
721
- });
722
- }
723
- get(key) {
724
- const existingEntry = this.findExistingEntry(key);
725
- return existingEntry?.[1];
726
- }
727
- set(key, value) {
728
- const existingKey = this.findExistingEntry(key);
729
- if (existingKey) {
730
- return super.set(existingKey?.[0], value);
731
- } else {
732
- return super.set(key, value);
733
- }
734
- }
735
- };
736
- var StreamMetadata = class {
737
- metadataByBreadcrumb;
738
- constructor() {
739
- this.metadataByBreadcrumb = new MetadataMap();
740
- }
741
- fromCatalogStreamMetadata(metadataArr) {
742
- metadataArr.forEach((metadata) => {
743
- this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
744
- });
745
- return this;
746
- }
747
- toString() {
748
- return this.metadataByBreadcrumb;
749
- }
750
- get(breadcrumb) {
751
- return this.metadataByBreadcrumb.get(breadcrumb);
752
- }
753
- write(breadcrumb, key, value) {
754
- if (value === void 0) {
755
- throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
756
-
757
- Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
758
- }
759
- if (!this.metadataByBreadcrumb.get(breadcrumb)) {
760
- this.metadataByBreadcrumb.set(breadcrumb, {});
761
- }
762
- const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
763
- const newValue = {};
764
- newValue[key] = value;
765
- this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
766
- return this;
767
- }
768
- getRootBreadcrumbMetadata() {
769
- const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
770
- if (!rootBreadcrumbMetadata) {
771
- throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level.
772
-
773
- Is your catalog properly generated?`);
774
- }
775
- return rootBreadcrumbMetadata;
776
- }
777
- isStreamSelected() {
778
- const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
779
- if (rootBreadcrumbMetadata.selected === false) {
780
- return false;
781
- }
782
- if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
783
- return true;
784
- }
785
- return false;
786
- }
787
- getKeyProperties() {
788
- const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
789
- if (!rootBreadcrumbMetadata["table-key-properties"]) {
790
- return [];
791
- }
792
- return rootBreadcrumbMetadata["table-key-properties"];
793
- }
794
- getReplicationKey() {
795
- const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
796
- if (!rootBreadcrumbMetadata["replication-key"]) {
797
- return void 0;
798
- }
799
- return rootBreadcrumbMetadata["replication-key"];
800
- }
801
- getReplicationMethod() {
802
- const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
803
- if (!rootBreadcrumbMetadata["replication-method"]) {
804
- return void 0;
805
- }
806
- return rootBreadcrumbMetadata["replication-method"];
807
- }
808
- getForcedReplicationMethod() {
809
- const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
810
- if (!rootBreadcrumbMetadata["forced-replication-method"]) {
811
- return void 0;
812
- }
813
- return rootBreadcrumbMetadata["forced-replication-method"];
814
- }
815
- toList() {
816
- return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
817
- const metadata = this.metadataByBreadcrumb.get(breadcrumb);
818
- if (!metadata) {
819
- throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
820
- }
821
- return {
822
- breadcrumb,
823
- metadata
824
- };
825
- });
826
- }
827
- };
828
-
829
564
  // src/sdk/models/state.ts
830
- var import_moment = __toESM(require("moment"));
565
+ var import_dayjs = __toESM(require("dayjs"));
831
566
 
832
567
  // src/sdk/constants/date.ts
833
568
  var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
@@ -842,12 +577,12 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
842
577
  if (!newRkValue) {
843
578
  throw new Error((0, import_util.format)(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
844
579
  }
845
- let newRkValueMoment = (0, import_moment.default)(newRkValue);
580
+ let newRkValueMoment = (0, import_dayjs.default)(newRkValue);
846
581
  let prevRkValue;
847
582
  let prevRkValueMoment;
848
583
  if (isSorted) {
849
584
  prevRkValue = state["replicationKeyValue"];
850
- prevRkValueMoment = (0, import_moment.default)(prevRkValue);
585
+ prevRkValueMoment = (0, import_dayjs.default)(prevRkValue);
851
586
  if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
852
587
  throw new Error(
853
588
  `Unsorted data detected in stream. Latest value '${newRkValue}' is
@@ -862,24 +597,24 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
862
597
  state[PROGRESS_MARKERS] = marker;
863
598
  }
864
599
  prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
865
- prevRkValueMoment = (0, import_moment.default)(prevRkValue || state["replicationKeyValue"]);
600
+ prevRkValueMoment = (0, import_dayjs.default)(prevRkValue || state["replicationKeyValue"]);
866
601
  if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
867
602
  newRkValueMoment = prevRkValueMoment;
868
603
  newRkValue = prevRkValue;
869
604
  }
870
605
  }
871
606
  const signpostMarker = state[SIGNPOST_MARKER];
872
- const replicationKeySignpostMoment = (0, import_moment.default)(signpostMarker);
607
+ const replicationKeySignpostMoment = (0, import_dayjs.default)(signpostMarker);
873
608
  if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
874
609
  newRkValueMoment = replicationKeySignpostMoment;
875
610
  newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
876
611
  }
877
612
  if (isSorted === false) {
878
613
  state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
879
- state[PROGRESS_MARKERS]["replicationKeyValue"] = (0, import_moment.default)(newRkValueMoment).format(defaultDateTimeFormat);
614
+ state[PROGRESS_MARKERS]["replicationKeyValue"] = (0, import_dayjs.default)(newRkValueMoment).format(defaultDateTimeFormat);
880
615
  } else {
881
616
  state["replicationKey"] = replicationKey;
882
- state["replicationKeyValue"] = (0, import_moment.default)(newRkValueMoment).format(defaultDateTimeFormat);
617
+ state["replicationKeyValue"] = (0, import_dayjs.default)(newRkValueMoment).format(defaultDateTimeFormat);
883
618
  }
884
619
  return state;
885
620
  };
@@ -901,7 +636,7 @@ var finalizeStateProgressMarkers = (state) => {
901
636
  return state;
902
637
  };
903
638
  var _greaterThan = (aValue, bValue) => {
904
- return (0, import_moment.default)(aValue).isAfter((0, import_moment.default)(bValue));
639
+ return (0, import_dayjs.default)(aValue).isAfter((0, import_dayjs.default)(bValue));
905
640
  };
906
641
  var extractStateForStream = (state, streamId) => {
907
642
  if (!state.bookmarks) {
@@ -924,26 +659,16 @@ var StateService = class {
924
659
  }
925
660
  return this.instance;
926
661
  }
927
- // To be deprecated
928
- setBookmark(streamId, ts) {
929
- this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
930
- }
931
- setBookmarkV2(streamId, streamState) {
662
+ setBookmark(streamId, streamState) {
932
663
  this.bookmarks[streamId] = streamState;
933
664
  }
934
- setBookmarkSignpostV2(streamId, signpostValue) {
665
+ setBookmarkSignpost(streamId, signpostValue) {
935
666
  if (!this.bookmarks[streamId]) {
936
667
  this.bookmarks[streamId] = {};
937
668
  }
938
669
  this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
939
670
  }
940
671
  getBookmark(streamId) {
941
- if (!this.bookmarks[streamId]) {
942
- return (0, import_moment.default)(0);
943
- }
944
- return (0, import_moment.default)(this.bookmarks[streamId]);
945
- }
946
- getBookmarkV2(streamId) {
947
672
  if (this.bookmarks[streamId]) {
948
673
  return this.bookmarks[streamId];
949
674
  } else {
@@ -983,7 +708,7 @@ var import_axios2 = __toESM(require("axios"));
983
708
  var qs2 = __toESM(require("qs"));
984
709
 
985
710
  // src/sdk/models/tap/stream.ts
986
- var import_moment2 = __toESM(require("moment"));
711
+ var import_dayjs2 = __toESM(require("dayjs"));
987
712
 
988
713
  // src/sdk/helpers/typing.ts
989
714
  var isDatetimeType = (typeDef) => {
@@ -1006,10 +731,11 @@ var isDatetimeType = (typeDef) => {
1006
731
  };
1007
732
 
1008
733
  // src/sdk/models/tap/stream.ts
1009
- var import_cloneDeep = __toESM(require("lodash/cloneDeep.js"));
734
+ var import_cloneDeep2 = __toESM(require("lodash/cloneDeep.js"));
1010
735
  var import_bluebird2 = __toESM(require("bluebird"));
1011
736
 
1012
737
  // src/sdk/models/tap/tap.ts
738
+ var import_cloneDeep = __toESM(require("lodash/cloneDeep.js"));
1013
739
  var import_bluebird = __toESM(require("bluebird"));
1014
740
  var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
1015
741
  var Tap = class {
@@ -1017,12 +743,21 @@ var Tap = class {
1017
743
  config;
1018
744
  streams;
1019
745
  concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
1020
- constructor(target, config) {
746
+ stateProvider;
747
+ tapState;
748
+ constructor(target, config, stateProvider) {
1021
749
  this.streams = [];
1022
750
  this.target = target;
1023
751
  this.config = config;
752
+ this.stateProvider = stateProvider;
753
+ this.tapState = { bookmarks: {} };
1024
754
  }
1025
755
  sync = async (options) => {
756
+ const initialState = await this.stateProvider.getState();
757
+ if (initialState.state && initialState.state.bookmarks) {
758
+ this.tapState = (0, import_cloneDeep.default)(initialState.state);
759
+ }
760
+ await this.init();
1026
761
  logger.info(`\u{1F680} Start syncing`);
1027
762
  const state = StateService.getInstance().get();
1028
763
  logger.info(`\u{1F4CD} Received state: %j`, state);
@@ -1059,9 +794,7 @@ var Stream = class {
1059
794
  config;
1060
795
  tapState;
1061
796
  target;
1062
- // Compile time values, can be overriden
1063
- // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
1064
- forcedReplicationMethod = void 0;
797
+ replicationMethod = void 0;
1065
798
  selectedByDefault = true;
1066
799
  STATE_MSG_FREQUENCY = 10;
1067
800
  streamId = "default";
@@ -1069,10 +802,9 @@ var Stream = class {
1069
802
  displayLabel = void 0;
1070
803
  description = void 0;
1071
804
  primaryKey = [];
1072
- forcedReplicationKey = void 0;
805
+ replicationKey = void 0;
1073
806
  // When set, use the state from another stream for this stream
1074
807
  useStateFromStreamId = void 0;
1075
- relationships = [];
1076
808
  children = [];
1077
809
  // If true, it means that the records in this stream are sorted by replicationKey.
1078
810
  // If false, then the records are coming in an unsorted manner.
@@ -1084,9 +816,9 @@ var Stream = class {
1084
816
  // Metrics configuration
1085
817
  rowsSyncedMetricsConf;
1086
818
  executionTimeMetricsConf;
1087
- constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
819
+ constructor(config, tapState, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
1088
820
  this.config = config;
1089
- this.tapState = (0, import_cloneDeep.default)(state);
821
+ this.tapState = (0, import_cloneDeep2.default)(tapState);
1090
822
  this.target = target;
1091
823
  this.childConcurrency = childConcurrency;
1092
824
  this.rowsSyncedMetricsConf = {
@@ -1159,13 +891,11 @@ var Stream = class {
1159
891
  if (!this.isSilent) {
1160
892
  const schema = await this.getSchema();
1161
893
  if (schema !== void 0) {
1162
- const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
1163
894
  const replicationConfig = this.configuredReplicationConfig();
1164
895
  const message = new SchemaMessage({
1165
896
  keyProperties: this.primaryKey,
1166
897
  stream: this.streamId,
1167
898
  schema: schema.jsonSchema,
1168
- relationships: relatedRels,
1169
899
  ...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
1170
900
  ...this.displayLabel ? { displayLabel: this.displayLabel } : {},
1171
901
  ...this.description ? { description: this.description } : {},
@@ -1185,7 +915,7 @@ var Stream = class {
1185
915
  const signpostMoment = await this.getReplicationKeySignpost();
1186
916
  const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
1187
917
  if (signpostValue) {
1188
- StateService.getInstance().setBookmarkSignpostV2(this.streamId, signpostValue);
918
+ StateService.getInstance().setBookmarkSignpost(this.streamId, signpostValue);
1189
919
  }
1190
920
  };
1191
921
  /**
@@ -1201,7 +931,7 @@ var Stream = class {
1201
931
  if (this.useStateFromStreamId) {
1202
932
  return;
1203
933
  }
1204
- if (this.forcedReplicationMethod && !this.forcedReplicationKey) {
934
+ if (this.replicationMethod && !this.replicationKey) {
1205
935
  return;
1206
936
  }
1207
937
  if (!replicationConfig.replicationKey) {
@@ -1211,14 +941,14 @@ var Stream = class {
1211
941
  );
1212
942
  }
1213
943
  const stateServiceInst = StateService.getInstance();
1214
- const state = stateServiceInst.getBookmarkV2(this.streamId);
944
+ const state = stateServiceInst.getBookmark(this.streamId);
1215
945
  const newState = incrementStreamState(
1216
946
  state,
1217
947
  replicationConfig.replicationKey,
1218
948
  latestRecord,
1219
949
  this.isSorted
1220
950
  );
1221
- StateService.getInstance().setBookmarkV2(this.streamId, newState);
951
+ StateService.getInstance().setBookmark(this.streamId, newState);
1222
952
  }
1223
953
  }
1224
954
  };
@@ -1258,19 +988,19 @@ var Stream = class {
1258
988
  this.replicationKey: ${replicationConfig.replicationKey}`);
1259
989
  if (state.replicationKeyValue) {
1260
990
  logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
1261
- return (0, import_moment2.default)(state.replicationKeyValue);
991
+ return (0, import_dayjs2.default)(state.replicationKeyValue);
1262
992
  } else if (startDate) {
1263
993
  logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
1264
- return (0, import_moment2.default)(startDate);
994
+ return (0, import_dayjs2.default)(startDate);
1265
995
  } else {
1266
996
  logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
1267
- return (0, import_moment2.default)(0);
997
+ return (0, import_dayjs2.default)(0);
1268
998
  }
1269
999
  }
1270
1000
  /**
1271
1001
  * Return the max allowable bookmark value for this stream's replication key.
1272
1002
  *
1273
- * For timestamp-based replication keys, this defaults to `moment()`. For
1003
+ * For timestamp-based replication keys, this defaults to `dayjs()`. For
1274
1004
  * non-timestamp replication keys, default to `undefined`.
1275
1005
  *
1276
1006
  * Override this value to prevent bookmarks from being advanced in cases where we
@@ -1278,7 +1008,7 @@ var Stream = class {
1278
1008
  */
1279
1009
  getReplicationKeySignpost = async () => {
1280
1010
  if (await this.isTimestampReplicationKey()) {
1281
- return (0, import_moment2.default)();
1011
+ return (0, import_dayjs2.default)();
1282
1012
  }
1283
1013
  return void 0;
1284
1014
  };
@@ -1286,22 +1016,22 @@ var Stream = class {
1286
1016
  * Return the default replication method for the stream that will be used to write the catalog.
1287
1017
  */
1288
1018
  defaultReplicationConfig = () => {
1289
- if (this.forcedReplicationMethod) {
1019
+ if (this.replicationMethod) {
1290
1020
  return {
1291
- replicationMethod: this.forcedReplicationMethod,
1292
- replicationKey: this.forcedReplicationKey,
1021
+ replicationMethod: this.replicationMethod,
1022
+ replicationKey: this.replicationKey,
1293
1023
  isForced: true
1294
1024
  };
1295
1025
  }
1296
- if (this.forcedReplicationKey || this.useStateFromStreamId) {
1026
+ if (this.replicationKey || this.useStateFromStreamId) {
1297
1027
  return {
1298
- replicationMethod: "INCREMENTAL",
1299
- replicationKey: this.forcedReplicationKey,
1028
+ replicationMethod: "INCREMENTAL" /* INCREMENTAL */,
1029
+ replicationKey: this.replicationKey,
1300
1030
  isForced: true
1301
1031
  };
1302
1032
  }
1303
1033
  return {
1304
- replicationMethod: "FULL_TABLE",
1034
+ replicationMethod: "FULL_TABLE" /* FULL_TABLE */,
1305
1035
  replicationKey: void 0,
1306
1036
  isForced: false
1307
1037
  };
@@ -1362,9 +1092,9 @@ var Stream = class {
1362
1092
  rowsSent += 1;
1363
1093
  }
1364
1094
  const stateServiceInst = StateService.getInstance();
1365
- const state = stateServiceInst.getBookmarkV2(this.streamId);
1095
+ const state = stateServiceInst.getBookmark(this.streamId);
1366
1096
  const newState = finalizeStateProgressMarkers(state);
1367
- stateServiceInst.setBookmarkV2(this.streamId, newState);
1097
+ stateServiceInst.setBookmark(this.streamId, newState);
1368
1098
  if (!parent) {
1369
1099
  logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
1370
1100
  } else {
@@ -1470,7 +1200,7 @@ function getTruncatedParamsForLog(params) {
1470
1200
  }
1471
1201
 
1472
1202
  // src/sdk/models/tap/restStream.ts
1473
- var import_cloneDeep2 = __toESM(require("lodash/cloneDeep.js"));
1203
+ var import_cloneDeep3 = __toESM(require("lodash/cloneDeep.js"));
1474
1204
  var import_isEqual = __toESM(require("lodash/isEqual.js"));
1475
1205
  var RESTStream = class extends Stream {
1476
1206
  axiosInstance;
@@ -1491,13 +1221,24 @@ var RESTStream = class extends Stream {
1491
1221
  */
1492
1222
  _prepareRequest = async (nextPageToken, parent) => {
1493
1223
  const method = this.httpMethod;
1494
- const url = this.getUrl(parent);
1224
+ const url = this.getNextUrl(nextPageToken, parent);
1495
1225
  if (!url) {
1496
1226
  throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
1497
1227
  }
1498
1228
  const authenticator = this.authenticator;
1229
+ let finalUrl = url;
1230
+ let urlQueryParams = {};
1231
+ const queryIndex = url.indexOf("?");
1232
+ if (queryIndex !== -1) {
1233
+ finalUrl = url.substring(0, queryIndex);
1234
+ const rawQueryString = url.substring(queryIndex + 1);
1235
+ urlQueryParams = qs2.parse(rawQueryString);
1236
+ }
1499
1237
  const params = {
1500
1238
  ...this.getNextUrlParams(nextPageToken, parent),
1239
+ // Query params embedded in URL should override computed pagination params
1240
+ // but should not override auth params which are appended last
1241
+ ...urlQueryParams,
1501
1242
  ...await authenticator?.getAuthQS(this.config)
1502
1243
  };
1503
1244
  const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
@@ -1516,7 +1257,7 @@ var RESTStream = class extends Stream {
1516
1257
  return qs2.stringify(params2, { arrayFormat: "repeat" });
1517
1258
  };
1518
1259
  const request = {
1519
- url,
1260
+ url: finalUrl,
1520
1261
  method,
1521
1262
  data: requestBody,
1522
1263
  params,
@@ -1589,7 +1330,7 @@ var RESTStream = class extends Stream {
1589
1330
  for await (const row of rows) {
1590
1331
  yield row;
1591
1332
  }
1592
- const previousToken = (0, import_cloneDeep2.default)(nextPageToken);
1333
+ const previousToken = (0, import_cloneDeep3.default)(nextPageToken);
1593
1334
  nextPageToken = this.getNextPageToken(resp, previousToken);
1594
1335
  if (nextPageToken && (0, import_isEqual.default)(nextPageToken, previousToken)) {
1595
1336
  throw new Error(
@@ -1668,7 +1409,7 @@ var RESTStream = class extends Stream {
1668
1409
  * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
1669
1410
  * @returns
1670
1411
  */
1671
- getUrl(parent) {
1412
+ getNextUrl(previousToken, parent) {
1672
1413
  const urlPattern = [this.baseUrl, this.path].join("");
1673
1414
  return urlPattern;
1674
1415
  }
@@ -2020,18 +1761,18 @@ var flattenSchema = (streamId, schema) => {
2020
1761
 
2021
1762
  // src/sdk/models/target/target.ts
2022
1763
  var import_jsonschema = require("jsonschema");
2023
- var import_moment4 = __toESM(require("moment"));
1764
+ var import_dayjs4 = __toESM(require("dayjs"));
2024
1765
 
2025
1766
  // src/sdk/models/target/streamDbState.ts
2026
- var import_moment3 = __toESM(require("moment"));
1767
+ var import_dayjs3 = __toESM(require("dayjs"));
2027
1768
 
2028
1769
  // src/sdk/models/target/temporaryFile.ts
2029
- var import_fs_extra2 = __toESM(require("fs-extra"));
1770
+ var import_fs_extra = __toESM(require("fs-extra"));
2030
1771
  var import_uniqueId = __toESM(require("lodash/uniqueId.js"));
2031
1772
  var createTemporaryFileStream = (streamId) => {
2032
- import_fs_extra2.default.ensureDirSync("./tmp");
1773
+ import_fs_extra.default.ensureDirSync("./tmp");
2033
1774
  const path = `./tmp/${safePath(streamId)}-${(0, import_uniqueId.default)()}.tmp`;
2034
- const writeStream = import_fs_extra2.default.createWriteStream(path, { flags: "w" });
1775
+ const writeStream = import_fs_extra.default.createWriteStream(path, { flags: "w" });
2035
1776
  return {
2036
1777
  stream: writeStream,
2037
1778
  path
@@ -2079,7 +1820,7 @@ function replaceSymbols(str) {
2079
1820
  }
2080
1821
 
2081
1822
  // src/sdk/models/target/streamDbState.ts
2082
- var fs4 = __toESM(require("fs"));
1823
+ var fs3 = __toESM(require("fs"));
2083
1824
  var StreamState = class {
2084
1825
  streamId;
2085
1826
  schema;
@@ -2104,7 +1845,7 @@ var StreamState = class {
2104
1845
  this.fileToLoad = createTemporaryFileStream(streamId);
2105
1846
  this.syncedRowCount = 0;
2106
1847
  this.keyProperties = [];
2107
- this.batchDate = (0, import_moment3.default)();
1848
+ this.batchDate = (0, import_dayjs3.default)();
2108
1849
  this.replicationMethod = replicationMethod;
2109
1850
  this._hasBeenLoadedYetDuringThisSync = false;
2110
1851
  }
@@ -2129,7 +1870,7 @@ var StreamState = class {
2129
1870
  resetFileToLoad() {
2130
1871
  this.fileToLoad.stream.close();
2131
1872
  const oldPath = this.fileToLoad.path;
2132
- fs4.unlinkSync(oldPath);
1873
+ fs3.unlinkSync(oldPath);
2133
1874
  this.fileToLoad = createTemporaryFileStream(this.streamId);
2134
1875
  this.batchedRowCount = 0;
2135
1876
  }
@@ -2160,8 +1901,8 @@ var semaphore = new import_async_mutex.Semaphore(1);
2160
1901
  var ITarget = class _ITarget {
2161
1902
  config;
2162
1903
  schemaHooks;
2163
- resolver;
2164
1904
  syncTime;
1905
+ stateProvider;
2165
1906
  // Latest state message received from the Tap
2166
1907
  // Not yet flushed as we didn't upload the stream data since receiving it
2167
1908
  batchedState;
@@ -2172,15 +1913,15 @@ var ITarget = class _ITarget {
2172
1913
  streams;
2173
1914
  // JSON Schema validator
2174
1915
  validator;
2175
- constructor(config, resolver) {
1916
+ constructor(config, stateProvider) {
2176
1917
  this.config = config;
2177
- this.resolver = resolver;
2178
- this.schemaHooks = resolver.getSchemaHooks();
1918
+ this.stateProvider = stateProvider;
1919
+ this.schemaHooks = [];
2179
1920
  this.streams = {};
2180
1921
  this.batchedState = {};
2181
1922
  this.flushedState = {};
2182
1923
  this.validator = new import_jsonschema.Validator();
2183
- this.syncTime = (0, import_moment4.default)();
1924
+ this.syncTime = (0, import_dayjs4.default)();
2184
1925
  this.renameColumnStore = new RenameColumnStore();
2185
1926
  }
2186
1927
  ///////////////////////////////////////////
@@ -2289,7 +2030,7 @@ var ITarget = class _ITarget {
2289
2030
  this.streams[streamId] = new StreamState(
2290
2031
  streamId,
2291
2032
  dbSyncInstance,
2292
- replicationMethod || "FULL_TABLE"
2033
+ replicationMethod || "FULL_TABLE" /* FULL_TABLE */
2293
2034
  );
2294
2035
  }
2295
2036
  replicationMethod = (message) => {
@@ -2366,39 +2107,7 @@ var ITarget = class _ITarget {
2366
2107
  return translation;
2367
2108
  }
2368
2109
  return k;
2369
- }),
2370
- relationships: {
2371
- left: message.relationships.left.flatMap((l) => {
2372
- if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
2373
- const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
2374
- const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
2375
- if (from && to) {
2376
- return [{
2377
- type: l.type,
2378
- streamId: l.streamId,
2379
- from,
2380
- to
2381
- }];
2382
- }
2383
- }
2384
- return [];
2385
- }),
2386
- right: message.relationships.right.flatMap((r) => {
2387
- if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
2388
- const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
2389
- const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
2390
- if (from && to) {
2391
- return [{
2392
- type: r.type,
2393
- streamId: r.streamId,
2394
- from,
2395
- to
2396
- }];
2397
- }
2398
- }
2399
- return [];
2400
- })
2401
- }
2110
+ })
2402
2111
  };
2403
2112
  await import_bluebird3.default.map(this.schemaHooks, async (hook) => {
2404
2113
  const input = {
@@ -2431,9 +2140,6 @@ var ITarget = class _ITarget {
2431
2140
  try {
2432
2141
  logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
2433
2142
  await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
2434
- const configResolver = loadResolver();
2435
- await configResolver.markSyncComplete();
2436
- await configResolver.flushMetrics();
2437
2143
  return Promise.resolve();
2438
2144
  } catch (err) {
2439
2145
  logger.error(`Error while handling complete event.`);
@@ -2476,8 +2182,7 @@ var ITarget = class _ITarget {
2476
2182
  }
2477
2183
  };
2478
2184
  emitState = (state) => {
2479
- const configResolver = loadResolver();
2480
- return writeStateFile(configResolver, JSON.stringify(state));
2185
+ return this.stateProvider.writeState(JSON.stringify(state));
2481
2186
  };
2482
2187
  uploadStreamsToWarehouse = async (streamsToUpload) => {
2483
2188
  try {
@@ -2549,15 +2254,15 @@ var safeTableName = (key) => {
2549
2254
 
2550
2255
  // src/targets/bigquery/service/record.ts
2551
2256
  var import_decimal = __toESM(require("decimal.js"));
2552
- var import_moment5 = __toESM(require("moment"));
2257
+ var import_dayjs5 = __toESM(require("dayjs"));
2553
2258
  var validateDateRange = (record, schema) => {
2554
2259
  Object.keys(schema).forEach((key) => {
2555
- const { format: format3 } = schema[key];
2556
- if (format3 === "date-time") {
2260
+ const { format: format6 } = schema[key];
2261
+ if (format6 === "date-time") {
2557
2262
  if (record[key]) {
2558
- const formattedDate = (0, import_moment5.default)(record[key]).format(defaultDateTimeFormat);
2559
- const isNotTooMuchInTheFuture = (0, import_moment5.default)(record[key]).isBefore((0, import_moment5.default)("9999-12-31 23:59:59.999999"));
2560
- const isNotTooMuchInThePast = (0, import_moment5.default)(record[key]).isAfter((0, import_moment5.default)("0001-01-01 00:00:00"));
2263
+ const formattedDate = (0, import_dayjs5.default)(record[key]).format(defaultDateTimeFormat);
2264
+ const isNotTooMuchInTheFuture = (0, import_dayjs5.default)(record[key]).isBefore((0, import_dayjs5.default)("9999-12-31 23:59:59.999999"));
2265
+ const isNotTooMuchInThePast = (0, import_dayjs5.default)(record[key]).isAfter((0, import_dayjs5.default)("0001-01-01 00:00:00"));
2561
2266
  if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
2562
2267
  record[key] = formattedDate;
2563
2268
  } else {
@@ -2612,7 +2317,7 @@ var convertNumberIntoDecimal = (record, schema) => {
2612
2317
  };
2613
2318
 
2614
2319
  // src/targets/bigquery/service/dbSync.ts
2615
- var import_moment6 = __toESM(require("moment"));
2320
+ var import_dayjs6 = __toESM(require("dayjs"));
2616
2321
 
2617
2322
  // src/targets/bigquery/service/bigquery.ts
2618
2323
  var import_bigquery = require("@google-cloud/bigquery");
@@ -2693,13 +2398,13 @@ function v4(options, buf, offset) {
2693
2398
  var v4_default = v4;
2694
2399
 
2695
2400
  // src/targets/bigquery/service/cloudStorage.ts
2696
- var uploadFileInBucket = async (streamId, gcsBuckerName, filePath) => {
2401
+ var uploadFileInBucket = async (streamId, gcsBuckerName, connectorId, filePath) => {
2697
2402
  try {
2698
2403
  const retryOptions = { autoRetry: true, maxRetries: 20 };
2699
2404
  let storage = new import_storage.Storage({ retryOptions });
2700
2405
  const bucketRef = storage.bucket(gcsBuckerName);
2701
2406
  await bucketRef.get({ autoCreate: false });
2702
- const destinationFileName = `${streamId}-${v4_default()}.jsonnl`;
2407
+ const destinationFileName = `${connectorId}/${streamId}-${v4_default()}.jsonnl`;
2703
2408
  await bucketRef.upload(filePath, {
2704
2409
  destination: destinationFileName
2705
2410
  });
@@ -2767,21 +2472,21 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
2767
2472
  safeColumnName = safeColumnName;
2768
2473
  getWarehouseTypeFromJSONSchema = (definition) => {
2769
2474
  const type = definition.type;
2770
- const format3 = definition.format;
2475
+ const format6 = definition.format;
2771
2476
  let typeArr;
2772
2477
  if (type instanceof Array) {
2773
2478
  typeArr = type;
2774
2479
  } else {
2775
2480
  typeArr = [type];
2776
2481
  }
2777
- if (format3) {
2778
- switch (format3) {
2482
+ if (format6) {
2483
+ switch (format6) {
2779
2484
  case "date-time":
2780
2485
  return "TIMESTAMP";
2781
2486
  case "json":
2782
2487
  return "STRING";
2783
2488
  default:
2784
- throw new Error(`StreamId: ${this.streamId} - Unsupported format: ${format3}`);
2489
+ throw new Error(`StreamId: ${this.streamId} - Unsupported format: ${format6}`);
2785
2490
  }
2786
2491
  } else if (typeArr.includes("number")) {
2787
2492
  return "NUMERIC";
@@ -2981,7 +2686,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
2981
2686
  await this.deleteStagingArea();
2982
2687
  }
2983
2688
  const schema = this.generateBigquerySchema();
2984
- const expirationTime = (0, import_moment6.default)().add("1", "day").valueOf().toString();
2689
+ const expirationTime = (0, import_dayjs6.default)().add(1, "day").valueOf().toString();
2985
2690
  const createTableOptions = {
2986
2691
  schema,
2987
2692
  expirationTime
@@ -3004,6 +2709,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
3004
2709
  const file = await uploadFileInBucket(
3005
2710
  this.streamId,
3006
2711
  this.config.loading_deck_gcs_bucket_name,
2712
+ this.config.connector_id,
3007
2713
  localFilePath
3008
2714
  );
3009
2715
  const metadata = {
@@ -3115,8 +2821,8 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
3115
2821
 
3116
2822
  // src/targets/bigquery/main.ts
3117
2823
  var BigQueryTarget = class extends ITarget {
3118
- constructor(config, resolver) {
3119
- super(config, resolver);
2824
+ constructor(config, stateProvider) {
2825
+ super(config, stateProvider);
3120
2826
  this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
3121
2827
  }
3122
2828
  static requiredConfigKeys = [
@@ -3139,6 +2845,75 @@ var BigQueryTarget = class extends ITarget {
3139
2845
  validateDateRange = validateDateRange;
3140
2846
  convertNumberIntoDecimal = convertNumberIntoDecimal;
3141
2847
  };
2848
+
2849
+ // src/sdk/service/gcs.ts
2850
+ var import_storage2 = require("@google-cloud/storage");
2851
+ var import_util3 = require("util");
2852
+ var getStorage = () => new import_storage2.Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
2853
+ var readObjectAsString = async (bucketName, objectPath) => {
2854
+ try {
2855
+ const storage = getStorage();
2856
+ const bucketRef = storage.bucket(bucketName);
2857
+ await bucketRef.get({ autoCreate: false });
2858
+ const fileRef = bucketRef.file(objectPath);
2859
+ const [exists] = await fileRef.exists();
2860
+ if (!exists) {
2861
+ throw new Error(`GCS object not found: gs://${bucketName}/${objectPath}`);
2862
+ }
2863
+ const [contents] = await fileRef.download();
2864
+ return contents.toString("utf8");
2865
+ } catch (err) {
2866
+ throw new Error((0, import_util3.format)(`error reading GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
2867
+ }
2868
+ };
2869
+ var writeStringObject = async (bucketName, objectPath, contents) => {
2870
+ try {
2871
+ const storage = getStorage();
2872
+ const bucketRef = storage.bucket(bucketName);
2873
+ await bucketRef.get({ autoCreate: false });
2874
+ const fileRef = bucketRef.file(objectPath);
2875
+ await fileRef.save(contents, { contentType: "application/json" });
2876
+ logger.info(`\u{1F5F3} Uploaded object to gs://${bucketName}/${objectPath}`);
2877
+ } catch (err) {
2878
+ throw new Error((0, import_util3.format)(`error writing GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
2879
+ }
2880
+ };
2881
+
2882
+ // src/state-providers/gcs/main.ts
2883
+ var import_util4 = require("util");
2884
+ var GCSStateProvider = class {
2885
+ bucketName;
2886
+ connectorId;
2887
+ constructor(connectorId, bucketName) {
2888
+ this.connectorId = connectorId;
2889
+ this.bucketName = bucketName;
2890
+ }
2891
+ getObjectPath() {
2892
+ return `${this.connectorId}/state.json`;
2893
+ }
2894
+ async getState() {
2895
+ const objectPath = this.getObjectPath();
2896
+ try {
2897
+ logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
2898
+ const raw = await readObjectAsString(this.bucketName, objectPath);
2899
+ const parsed = JSON.parse(raw);
2900
+ return { state: parsed };
2901
+ } catch (err) {
2902
+ logger.warn((0, import_util4.format)(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
2903
+ return { state: void 0 };
2904
+ }
2905
+ }
2906
+ async writeState(state) {
2907
+ const objectPath = this.getObjectPath();
2908
+ try {
2909
+ logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
2910
+ await writeStringObject(this.bucketName, objectPath, state);
2911
+ } catch (err) {
2912
+ logger.error((0, import_util4.format)(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
2913
+ throw new Error((0, import_util4.format)(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
2914
+ }
2915
+ }
2916
+ };
3142
2917
  // Annotate the CommonJS export names for ESM import in node:
3143
2918
  0 && (module.exports = {
3144
2919
  ALL_STREAM_ID_KEYWORD,
@@ -3149,6 +2924,7 @@ var BigQueryTarget = class extends ITarget {
3149
2924
  CounterMetric,
3150
2925
  DEFAULT_MAX_CONCURRENT_STREAMS,
3151
2926
  EXECUTION_TIME_METRIC_NAME,
2927
+ GCSStateProvider,
3152
2928
  ITarget,
3153
2929
  MissingFieldInSchemaError,
3154
2930
  MissingSchemaError,
@@ -3156,14 +2932,13 @@ var BigQueryTarget = class extends ITarget {
3156
2932
  ROWS_SYNCED_METRIC_NAME,
3157
2933
  RecordMessage,
3158
2934
  RenameColumnStore,
2935
+ ReplicationMethod,
3159
2936
  ReplicationMethodMessage,
3160
- Resolver,
3161
2937
  SchemaMessage,
3162
2938
  SchemaValidationError,
3163
2939
  StateMessage,
3164
2940
  StateService,
3165
2941
  Stream,
3166
- StreamMetadata,
3167
2942
  StreamWarehouseSyncService,
3168
2943
  Tap,
3169
2944
  _greaterThan,
@@ -3171,7 +2946,6 @@ var BigQueryTarget = class extends ITarget {
3171
2946
  createTemporaryFileStream,
3172
2947
  extractStateForStream,
3173
2948
  finalizeStateProgressMarkers,
3174
- findRelatedRelationships,
3175
2949
  flattenSchema,
3176
2950
  getAllMetrics,
3177
2951
  getAxiosInstance,
@@ -3185,7 +2959,6 @@ var BigQueryTarget = class extends ITarget {
3185
2959
  haltAndCatchFire,
3186
2960
  incrementStreamState,
3187
2961
  loadJson,
3188
- loadResolver,
3189
2962
  loadSchema,
3190
2963
  logger,
3191
2964
  postFormDataApiCall,
@@ -3193,7 +2966,6 @@ var BigQueryTarget = class extends ITarget {
3193
2966
  postUrlEncodedApiCall,
3194
2967
  printMemoryFootprint,
3195
2968
  removeParasiteProperties,
3196
- safePath,
3197
- writeStateFile
2969
+ safePath
3198
2970
  });
3199
2971
  //# sourceMappingURL=index.js.map