@whaly/connector-sdk 0.1.1 → 0.2.0

package/dist/index.mjs

@@ -2,180 +2,10 @@
 import { readFileSync } from "fs";
 
 // src/sdk/service/logger.ts
-import * as winston2 from "winston";
-
-// src/sdk/models/resolver.ts
-var Resolver = class {
-  constructor() {
-  }
-};
-
-// src/resolvers/local-file/services/files.ts
-import fs from "fs";
-var writeFile = (path, content) => {
-  return Promise.resolve(fs.writeFileSync(path, content));
-};
-var readAndParseJSONFile = (path) => {
-  if (!fs.existsSync(path)) {
-    throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
-  }
-  const raw = fs.readFileSync(path);
-  const parsed = JSON.parse(raw.toString());
-  return parsed;
-};
-
-// src/resolvers/local-file/hook/schemaHook.ts
-import fsExtra from "fs-extra";
-var LocalSchemaHook = class {
-  async writeSchema(input) {
-    await fsExtra.appendFile("./schema.tmp", JSON.stringify(input) + "\n");
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-import winston from "winston";
-
-// src/sdk/service/metric.ts
-var metricStore = {};
-var ALL_STREAM_ID_KEYWORD = "all";
-var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-var API_CALLS_METRIC_NAME = "api_calls_count";
-var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-var getCounterMetrics = (name, streamIds) => {
-  return streamIds.map((streamId) => {
-    return getCounterMetric(name, streamId);
-  });
-};
-var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
-  if (!metricStore[streamId]) {
-    metricStore[streamId] = {};
-  }
-  const streamMetricStore = metricStore[streamId];
-  if (!streamMetricStore[name]) {
-    streamMetricStore[name] = new CounterMetric(name, streamId);
-  }
-  return streamMetricStore[name];
-};
-var getAllMetrics = () => {
-  return Object.keys(metricStore).flatMap((streamId) => {
-    const streamMetricStore = metricStore[streamId];
-    if (!streamMetricStore) {
-      return [];
-    }
-    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
-  });
-};
-var CounterMetric = class {
-  value = 0;
-  name;
-  streamId;
-  constructor(name, streamId) {
-    this.name = name;
-    this.streamId = streamId;
-  }
-  increment(inc = 1) {
-    this.value = this.value + inc;
-  }
-  getStreamId() {
-    return this.streamId;
-  }
-  getName() {
-    return this.name;
-  }
-  getValue() {
-    return this.value;
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
-var statePath = "./state.json";
-var metricsPath = "./metrics.json";
-var LocalFilesResolver = class extends Resolver {
-  checkIfCanSync() {
-    return Promise.resolve(true);
-  }
-  markSyncStarted() {
-    return Promise.resolve();
-  }
-  onError(errorType, errorText, errorDebugText) {
-    const path = `./error.json`;
-    return writeFile(path, JSON.stringify({
-      errorType,
-      errorText,
-      errorDebugText
-    }));
-  }
-  getSchemaHooks() {
-    const wlyHook = new LocalSchemaHook();
-    return [wlyHook];
-  }
-  writeSchema() {
-    return Promise.resolve();
-  }
-  constructor() {
-    super();
-  }
-  getState() {
-    logger.info(`\u{1F4C1} Using the LOCAL resolver`);
-    const state = readAndParseJSONFile(statePath);
-    return Promise.resolve({
-      state
-    });
-  }
-  writeCatalog(catalog) {
-    return Promise.resolve();
-  }
-  writeState(state) {
-    return writeFile(statePath, state);
-  }
-  markSyncFailed() {
-    logger.info(`\u{1F622} Sync has failed.`);
-    return Promise.resolve();
-  }
-  markSyncComplete() {
-    logger.info(`\u{1F389} Sync is complete.`);
-    return Promise.resolve();
-  }
-  markDiscoveryComplete() {
-    logger.info(`\u{1F389} Discovery is complete.`);
-    return Promise.resolve();
-  }
-  async flushMetrics() {
-    const allMetrics = getAllMetrics();
-    const metricsOuput = allMetrics.map((metric) => {
-      return {
-        streamId: metric.getStreamId(),
-        name: metric.getName(),
-        value: metric.getValue()
-      };
-    }).map((obj) => JSON.stringify(obj)).join(`
-`);
-    return writeFile(metricsPath, metricsOuput);
-  }
-  updateSourceValue(optionKey, optionValue) {
-    logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
-    return Promise.resolve();
-  }
-  getLogFormat() {
-    return combine(
-      timestamp(),
-      errors({ stack: true }),
-      splat(),
-      prettyPrint()
-    );
-  }
-};
-
-// src/sdk/service/resolver.ts
-var loadResolver = () => {
-  return new LocalFilesResolver();
-};
-
-// src/sdk/service/logger.ts
+import * as winston from "winston";
 var BATCH_INTERVAL_MS = 100;
 var transports2 = {
-  console: new winston2.transports.Console({
+  console: new winston.transports.Console({
     stderrLevels: [],
     level: "info"
   })
@@ -185,8 +15,14 @@ var getTransports = () => {
     transports2.console
   ];
 };
-var winstonLogger = winston2.createLogger({
-  format: loadResolver().getLogFormat(),
+var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
+var winstonLogger = winston.createLogger({
+  format: combine(
+    timestamp(),
+    errors({ stack: true }),
+    splat(),
+    prettyPrint()
+  ),
   transports: getTransports()
 });
 var isWinstonOpen = true;
@@ -215,10 +51,6 @@ var logger = {
 };
 
 // src/sdk/utils.ts
-var writeStateFile = (resolver, state) => {
-  logger.info(`\u{1F4DD} Writing the state file.`);
-  return resolver.writeState(state);
-};
 function readFile(fileName, filePath) {
   try {
     return readFileSync(filePath);
@@ -248,7 +80,7 @@ var loadSchema = (streamId) => {
 // src/sdk/service/network.ts
 import axios from "axios";
 import { URLSearchParams } from "url";
-import * as fs2 from "fs";
+import * as fs from "fs";
 import * as qs from "qs";
 import axiosRetry, { isNetworkOrIdempotentRequestError } from "axios-retry";
 var shouldRetry = (error) => {
@@ -361,7 +193,7 @@ var getJSONApiCall = async (endpoint, config, privateLogging) => {
 };
 var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
   const instance = getAxiosInstance(config.retryCount);
-  const writer = fs2.createWriteStream(output);
+  const writer = fs.createWriteStream(output);
   const paramsSerializer = (params) => {
     return qs.stringify(params, { arrayFormat: "repeat" });
   };
@@ -455,6 +287,58 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
   }
 };
 
+// src/sdk/service/metric.ts
+var metricStore = {};
+var ALL_STREAM_ID_KEYWORD = "all";
+var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+var API_CALLS_METRIC_NAME = "api_calls_count";
+var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+var getCounterMetrics = (name, streamIds) => {
+  return streamIds.map((streamId) => {
+    return getCounterMetric(name, streamId);
+  });
+};
+var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
+  if (!metricStore[streamId]) {
+    metricStore[streamId] = {};
+  }
+  const streamMetricStore = metricStore[streamId];
+  if (!streamMetricStore[name]) {
+    streamMetricStore[name] = new CounterMetric(name, streamId);
+  }
+  return streamMetricStore[name];
+};
+var getAllMetrics = () => {
+  return Object.keys(metricStore).flatMap((streamId) => {
+    const streamMetricStore = metricStore[streamId];
+    if (!streamMetricStore) {
+      return [];
+    }
+    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
+  });
+};
+var CounterMetric = class {
+  value = 0;
+  name;
+  streamId;
+  constructor(name, streamId) {
+    this.name = name;
+    this.streamId = streamId;
+  }
+  increment(inc = 1) {
+    this.value = this.value + inc;
+  }
+  getStreamId() {
+    return this.streamId;
+  }
+  getName() {
+    return this.name;
+  }
+  getValue() {
+    return this.value;
+  }
+};
+
 // src/sdk/service/memory.ts
 var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
 var printMemoryFootprint = (prefix) => {
@@ -470,9 +354,6 @@ var printMemoryFootprint = (prefix) => {
 
 // src/sdk/service/exit.ts
 async function gracefulExit(exitCode) {
-  if (exitCode !== 0) {
-    await loadResolver().markSyncFailed();
-  }
   const loggerFinish = new Promise((resolve, reject) => {
     logger.closeWinston(resolve);
   });
@@ -492,34 +373,15 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
     errorText,
     errorDebugText
   );
-  const resolver = loadResolver();
-  await resolver.onError(errorType, errorText, errorDebugText);
   await gracefulExit(0);
 };
 
-// src/sdk/service/relationship.ts
-var findRelatedRelationships = (streamId, allRels) => {
-  return allRels.reduce((acc, rel) => {
-    const base = {
-      type: rel.type,
-      from: rel.from,
-      to: rel.to
-    };
-    if (rel.left === streamId) {
-      acc.right.push({
-        streamId: rel.right,
-        ...base
-      });
-    }
-    if (rel.right === streamId) {
-      acc.left.push({
-        streamId: rel.left,
-        ...base
-      });
-    }
-    return acc;
-  }, { left: [], right: [] });
-};
+// src/sdk/models/replication.ts
+var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
+  ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
+  ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
+  return ReplicationMethod2;
+})(ReplicationMethod || {});
 
 // src/sdk/models/messages.ts
 var RecordMessage = class {
@@ -579,7 +441,6 @@ var SchemaMessage = class {
   displayLabel;
   description;
   propertiesMetadata;
-  relationships;
   constructor(opts) {
     const {
       stream,
@@ -588,7 +449,6 @@ var SchemaMessage = class {
       bookmarkProperties,
       displayLabel,
       description,
-      relationships,
       propertiesMetadata
     } = opts;
     this.stream = stream;
@@ -597,7 +457,6 @@ var SchemaMessage = class {
     this.bookmarkProperties = bookmarkProperties;
     this.displayLabel = displayLabel;
     this.description = description;
-    this.relationships = relationships;
     this.propertiesMetadata = propertiesMetadata;
   }
   toString() {
@@ -607,8 +466,7 @@ var SchemaMessage = class {
       schema: this.schema,
       key_properties: this.keyProperties,
       label: this.displayLabel,
-      description: this.description,
-      relationships: this.relationships
+      description: this.description
     };
     if (this.bookmarkProperties) {
       result["bookmark_properties"] = this.bookmarkProperties;
@@ -617,148 +475,28 @@ var SchemaMessage = class {
   }
 };
 
-// src/sdk/models/metadata.ts
-var MetadataMap = class extends Map {
-  areEquals(array1, array2) {
-    return array1.length === array2.length && array1.every(function(value, index) {
-      return value === array2[index];
-    });
-  }
-  findExistingEntry(key) {
-    return Array.from(super.entries()).find((entry) => {
-      if (this.areEquals(entry[0], key)) {
-        return true;
-      }
-    });
-  }
-  get(key) {
-    const existingEntry = this.findExistingEntry(key);
-    return existingEntry?.[1];
-  }
-  set(key, value) {
-    const existingKey = this.findExistingEntry(key);
-    if (existingKey) {
-      return super.set(existingKey?.[0], value);
-    } else {
-      return super.set(key, value);
-    }
-  }
-};
-var StreamMetadata = class {
-  metadataByBreadcrumb;
-  constructor() {
-    this.metadataByBreadcrumb = new MetadataMap();
-  }
-  fromCatalogStreamMetadata(metadataArr) {
-    metadataArr.forEach((metadata) => {
-      this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
-    });
-    return this;
-  }
-  toString() {
-    return this.metadataByBreadcrumb;
-  }
-  get(breadcrumb) {
-    return this.metadataByBreadcrumb.get(breadcrumb);
-  }
-  write(breadcrumb, key, value) {
-    if (value === void 0) {
-      throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
-            
-            Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
-    }
-    if (!this.metadataByBreadcrumb.get(breadcrumb)) {
-      this.metadataByBreadcrumb.set(breadcrumb, {});
-    }
-    const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
-    const newValue = {};
-    newValue[key] = value;
-    this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
-    return this;
-  }
-  getRootBreadcrumbMetadata() {
-    const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
-    if (!rootBreadcrumbMetadata) {
-      throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level. 
-            
-            Is your catalog properly generated?`);
-    }
-    return rootBreadcrumbMetadata;
-  }
-  isStreamSelected() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (rootBreadcrumbMetadata.selected === false) {
-      return false;
-    }
-    if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
-      return true;
-    }
-    return false;
-  }
-  getKeyProperties() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["table-key-properties"]) {
-      return [];
-    }
-    return rootBreadcrumbMetadata["table-key-properties"];
-  }
-  getReplicationKey() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-key"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-key"];
-  }
-  getReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-method"];
-  }
-  getForcedReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["forced-replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["forced-replication-method"];
-  }
-  toList() {
-    return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
-      const metadata = this.metadataByBreadcrumb.get(breadcrumb);
-      if (!metadata) {
-        throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
-      }
-      return {
-        breadcrumb,
-        metadata
-      };
-    });
-  }
-};
-
 // src/sdk/models/state.ts
-import moment from "moment";
+import dayjs from "dayjs";
 
 // src/sdk/constants/date.ts
 var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
 
 // src/sdk/models/state.ts
-import { format } from "util";
+import { format as format2 } from "util";
 var PROGRESS_MARKERS = "progressMarkers";
 var PROGRESS_MARKER_NOTE = "Note";
 var SIGNPOST_MARKER = "replicationKeySignpost";
 var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
   let newRkValue = latestRecord[replicationKey];
   if (!newRkValue) {
-    throw new Error(format(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
+    throw new Error(format2(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
   }
-  let newRkValueMoment = moment(newRkValue);
+  let newRkValueMoment = dayjs(newRkValue);
   let prevRkValue;
   let prevRkValueMoment;
   if (isSorted) {
     prevRkValue = state["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue);
+    prevRkValueMoment = dayjs(prevRkValue);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       throw new Error(
         `Unsorted data detected in stream. Latest value '${newRkValue}' is
@@ -773,24 +511,24 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
       state[PROGRESS_MARKERS] = marker;
     }
     prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue || state["replicationKeyValue"]);
+    prevRkValueMoment = dayjs(prevRkValue || state["replicationKeyValue"]);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       newRkValueMoment = prevRkValueMoment;
       newRkValue = prevRkValue;
     }
   }
   const signpostMarker = state[SIGNPOST_MARKER];
-  const replicationKeySignpostMoment = moment(signpostMarker);
+  const replicationKeySignpostMoment = dayjs(signpostMarker);
   if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
     newRkValueMoment = replicationKeySignpostMoment;
     newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
   }
   if (isSorted === false) {
     state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
-    state[PROGRESS_MARKERS]["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state[PROGRESS_MARKERS]["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   } else {
     state["replicationKey"] = replicationKey;
-    state["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   }
   return state;
 };
@@ -812,7 +550,7 @@ var finalizeStateProgressMarkers = (state) => {
   return state;
 };
 var _greaterThan = (aValue, bValue) => {
-  return moment(aValue).isAfter(moment(bValue));
+  return dayjs(aValue).isAfter(dayjs(bValue));
 };
 var extractStateForStream = (state, streamId) => {
   if (!state.bookmarks) {
@@ -835,26 +573,16 @@ var StateService = class {
     }
     return this.instance;
   }
-  // To be deprecated
-  setBookmark(streamId, ts) {
-    this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
-  }
-  setBookmarkV2(streamId, streamState) {
+  setBookmark(streamId, streamState) {
     this.bookmarks[streamId] = streamState;
   }
-  setBookmarkSignpostV2(streamId, signpostValue) {
+  setBookmarkSignpost(streamId, signpostValue) {
     if (!this.bookmarks[streamId]) {
       this.bookmarks[streamId] = {};
     }
     this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
   }
   getBookmark(streamId) {
-    if (!this.bookmarks[streamId]) {
-      return moment(0);
-    }
-    return moment(this.bookmarks[streamId]);
-  }
-  getBookmarkV2(streamId) {
     if (this.bookmarks[streamId]) {
       return this.bookmarks[streamId];
     } else {
@@ -894,7 +622,7 @@ import axios2 from "axios";
 import * as qs2 from "qs";
 
 // src/sdk/models/tap/stream.ts
-import moment2 from "moment";
+import dayjs2 from "dayjs";
 
 // src/sdk/helpers/typing.ts
 var isDatetimeType = (typeDef) => {
@@ -917,10 +645,11 @@ var isDatetimeType = (typeDef) => {
 };
 
 // src/sdk/models/tap/stream.ts
-import cloneDeep from "lodash/cloneDeep.js";
+import cloneDeep2 from "lodash/cloneDeep.js";
 import Bluebird2 from "bluebird";
 
 // src/sdk/models/tap/tap.ts
+import cloneDeep from "lodash/cloneDeep.js";
 import Bluebird from "bluebird";
 var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
 var Tap = class {
@@ -928,12 +657,21 @@ var Tap = class {
   config;
   streams;
   concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
-  constructor(target, config) {
+  stateProvider;
+  tapState;
+  constructor(target, config, stateProvider) {
     this.streams = [];
     this.target = target;
     this.config = config;
+    this.stateProvider = stateProvider;
+    this.tapState = { bookmarks: {} };
   }
   sync = async (options) => {
+    const initialState = await this.stateProvider.getState();
+    if (initialState.state && initialState.state.bookmarks) {
+      this.tapState = cloneDeep(initialState.state);
+    }
+    await this.init();
     logger.info(`\u{1F680} Start syncing`);
     const state = StateService.getInstance().get();
     logger.info(`\u{1F4CD} Received state: %j`, state);
@@ -964,15 +702,13 @@ var Tap = class {
 };
 
 // src/sdk/models/tap/stream.ts
-import { format as format2 } from "util";
+import { format as format3 } from "util";
 var Stream = class {
   // Runtime values, can't be overriden
   config;
   tapState;
   target;
-  // Compile time values, can be overriden
-  // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
-  forcedReplicationMethod = void 0;
+  replicationMethod = void 0;
   selectedByDefault = true;
   STATE_MSG_FREQUENCY = 10;
   streamId = "default";
@@ -980,10 +716,9 @@ var Stream = class {
   displayLabel = void 0;
   description = void 0;
   primaryKey = [];
-  forcedReplicationKey = void 0;
+  replicationKey = void 0;
   // When set, use the state from another stream for this stream
   useStateFromStreamId = void 0;
-  relationships = [];
   children = [];
   // If true, it means that the records in this stream are sorted by replicationKey.
   // If false, then the records are coming in an unsorted manner.
@@ -995,9 +730,9 @@ var Stream = class {
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
-  constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+  constructor(config, tapState, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
     this.config = config;
-    this.tapState = cloneDeep(state);
+    this.tapState = cloneDeep2(tapState);
     this.target = target;
     this.childConcurrency = childConcurrency;
     this.rowsSyncedMetricsConf = {
@@ -1029,7 +764,7 @@ var Stream = class {
       return this.flush();
     } catch (err) {
       if (err instanceof Error) {
-        throw new Error(format2(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
+        throw new Error(format3(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
       }
       throw err;
     }
@@ -1070,13 +805,11 @@ var Stream = class {
     if (!this.isSilent) {
       const schema = await this.getSchema();
       if (schema !== void 0) {
-        const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
         const replicationConfig = this.configuredReplicationConfig();
         const message = new SchemaMessage({
           keyProperties: this.primaryKey,
           stream: this.streamId,
           schema: schema.jsonSchema,
-          relationships: relatedRels,
           ...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
           ...this.displayLabel ? { displayLabel: this.displayLabel } : {},
           ...this.description ? { description: this.description } : {},
@@ -1096,7 +829,7 @@ var Stream = class {
     const signpostMoment = await this.getReplicationKeySignpost();
     const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
     if (signpostValue) {
-      StateService.getInstance().setBookmarkSignpostV2(this.streamId, signpostValue);
+      StateService.getInstance().setBookmarkSignpost(this.streamId, signpostValue);
     }
   };
   /**
@@ -1112,7 +845,7 @@ var Stream = class {
         if (this.useStateFromStreamId) {
           return;
         }
-        if (this.forcedReplicationMethod && !this.forcedReplicationKey) {
+        if (this.replicationMethod && !this.replicationKey) {
           return;
         }
         if (!replicationConfig.replicationKey) {
@@ -1122,14 +855,14 @@ var Stream = class {
           );
         }
         const stateServiceInst = StateService.getInstance();
-        const state = stateServiceInst.getBookmarkV2(this.streamId);
+        const state = stateServiceInst.getBookmark(this.streamId);
         const newState = incrementStreamState(
           state,
           replicationConfig.replicationKey,
           latestRecord,
           this.isSorted
         );
-        StateService.getInstance().setBookmarkV2(this.streamId, newState);
+        StateService.getInstance().setBookmark(this.streamId, newState);
       }
     }
   };
@@ -1169,19 +902,19 @@ var Stream = class {
         this.replicationKey: ${replicationConfig.replicationKey}`);
     if (state.replicationKeyValue) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
-      return moment2(state.replicationKeyValue);
+      return dayjs2(state.replicationKeyValue);
     } else if (startDate) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
-      return moment2(startDate);
+      return dayjs2(startDate);
     } else {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
-      return moment2(0);
+      return dayjs2(0);
     }
   }
   /**
    * Return the max allowable bookmark value for this stream's replication key.
    * 
-   * For timestamp-based replication keys, this defaults to `moment()`. For
+   * For timestamp-based replication keys, this defaults to `dayjs()`. For
    * non-timestamp replication keys, default to `undefined`.
    * 
    * Override this value to prevent bookmarks from being advanced in cases where we
@@ -1189,7 +922,7 @@ var Stream = class {
    */
   getReplicationKeySignpost = async () => {
     if (await this.isTimestampReplicationKey()) {
-      return moment2();
+      return dayjs2();
     }
     return void 0;
   };
@@ -1197,22 +930,22 @@ var Stream = class {
    * Return the default replication method for the stream that will be used to write the catalog.
    */
   defaultReplicationConfig = () => {
-    if (this.forcedReplicationMethod) {
+    if (this.replicationMethod) {
       return {
-        replicationMethod: this.forcedReplicationMethod,
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: this.replicationMethod,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
-    if (this.forcedReplicationKey || this.useStateFromStreamId) {
+    if (this.replicationKey || this.useStateFromStreamId) {
       return {
-        replicationMethod: "INCREMENTAL",
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: "INCREMENTAL" /* INCREMENTAL */,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
     return {
-      replicationMethod: "FULL_TABLE",
+      replicationMethod: "FULL_TABLE" /* FULL_TABLE */,
       replicationKey: void 0,
       isForced: false
     };
@@ -1273,9 +1006,9 @@ var Stream = class {
       rowsSent += 1;
     }
     const stateServiceInst = StateService.getInstance();
-    const state = stateServiceInst.getBookmarkV2(this.streamId);
+    const state = stateServiceInst.getBookmark(this.streamId);
     const newState = finalizeStateProgressMarkers(state);
-    stateServiceInst.setBookmarkV2(this.streamId, newState);
+    stateServiceInst.setBookmark(this.streamId, newState);
     if (!parent) {
       logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
     } else {
@@ -1381,7 +1114,7 @@ function getTruncatedParamsForLog(params) {
 }
 
 // src/sdk/models/tap/restStream.ts
-import cloneDeep2 from "lodash/cloneDeep.js";
+import cloneDeep3 from "lodash/cloneDeep.js";
 import isEqual from "lodash/isEqual.js";
 var RESTStream = class extends Stream {
   axiosInstance;
@@ -1402,13 +1135,24 @@ var RESTStream = class extends Stream {
    */
   _prepareRequest = async (nextPageToken, parent) => {
     const method = this.httpMethod;
-    const url = this.getUrl(parent);
+    const url = this.getNextUrl(nextPageToken, parent);
     if (!url) {
       throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
     }
     const authenticator = this.authenticator;
+    let finalUrl = url;
+    let urlQueryParams = {};
+    const queryIndex = url.indexOf("?");
+    if (queryIndex !== -1) {
+      finalUrl = url.substring(0, queryIndex);
+      const rawQueryString = url.substring(queryIndex + 1);
+      urlQueryParams = qs2.parse(rawQueryString);
+    }
     const params = {
       ...this.getNextUrlParams(nextPageToken, parent),
+      // Query params embedded in URL should override computed pagination params
+      // but should not override auth params which are appended last
+      ...urlQueryParams,
       ...await authenticator?.getAuthQS(this.config)
     };
     const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
@@ -1427,7 +1171,7 @@ var RESTStream = class extends Stream {
       return qs2.stringify(params2, { arrayFormat: "repeat" });
     };
     const request = {
-      url,
+      url: finalUrl,
       method,
       data: requestBody,
       params,
@@ -1500,7 +1244,7 @@ var RESTStream = class extends Stream {
       for await (const row of rows) {
         yield row;
       }
-      const previousToken = cloneDeep2(nextPageToken);
+      const previousToken = cloneDeep3(nextPageToken);
       nextPageToken = this.getNextPageToken(resp, previousToken);
       if (nextPageToken && isEqual(nextPageToken, previousToken)) {
         throw new Error(
@@ -1579,7 +1323,7 @@ var RESTStream = class extends Stream {
    * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
    * @returns 
    */
-  getUrl(parent) {
+  getNextUrl(previousToken, parent) {
     const urlPattern = [this.baseUrl, this.path].join("");
     return urlPattern;
   }
@@ -1931,18 +1675,18 @@ var flattenSchema = (streamId, schema) => {
 
 // src/sdk/models/target/target.ts
 import { Validator } from "jsonschema";
-import moment4 from "moment";
+import dayjs4 from "dayjs";
 
 // src/sdk/models/target/streamDbState.ts
-import moment3 from "moment";
+import dayjs3 from "dayjs";
 
 // src/sdk/models/target/temporaryFile.ts
-import fs3 from "fs-extra";
+import fs2 from "fs-extra";
 import uniqueId from "lodash/uniqueId.js";
 var createTemporaryFileStream = (streamId) => {
-  fs3.ensureDirSync("./tmp");
+  fs2.ensureDirSync("./tmp");
   const path = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
-  const writeStream = fs3.createWriteStream(path, { flags: "w" });
+  const writeStream = fs2.createWriteStream(path, { flags: "w" });
   return {
     stream: writeStream,
     path
@@ -1990,7 +1734,7 @@ function replaceSymbols(str) {
 }
 
 // src/sdk/models/target/streamDbState.ts
-import * as fs4 from "fs";
+import * as fs3 from "fs";
 var StreamState = class {
   streamId;
   schema;
@@ -2015,7 +1759,7 @@ var StreamState = class {
     this.fileToLoad = createTemporaryFileStream(streamId);
     this.syncedRowCount = 0;
     this.keyProperties = [];
-    this.batchDate = moment3();
+    this.batchDate = dayjs3();
     this.replicationMethod = replicationMethod;
     this._hasBeenLoadedYetDuringThisSync = false;
   }
@@ -2040,7 +1784,7 @@ var StreamState = class {
   resetFileToLoad() {
     this.fileToLoad.stream.close();
     const oldPath = this.fileToLoad.path;
-    fs4.unlinkSync(oldPath);
+    fs3.unlinkSync(oldPath);
     this.fileToLoad = createTemporaryFileStream(this.streamId);
     this.batchedRowCount = 0;
   }
@@ -2071,8 +1815,8 @@ var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
   schemaHooks;
-  resolver;
   syncTime;
+  stateProvider;
   // Latest state message received from the Tap
   // Not yet flushed as we didn't upload the stream data since receiving it 
   batchedState;
@@ -2083,15 +1827,15 @@ var ITarget = class _ITarget {
   streams;
   // JSON Schema validator
   validator;
-  constructor(config, resolver) {
+  constructor(config, stateProvider) {
     this.config = config;
-    this.resolver = resolver;
-    this.schemaHooks = resolver.getSchemaHooks();
+    this.stateProvider = stateProvider;
+    this.schemaHooks = [];
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
     this.validator = new Validator();
-    this.syncTime = moment4();
+    this.syncTime = dayjs4();
     this.renameColumnStore = new RenameColumnStore();
   }
   ///////////////////////////////////////////
@@ -2200,7 +1944,7 @@ var ITarget = class _ITarget {
     this.streams[streamId] = new StreamState(
       streamId,
       dbSyncInstance,
-      replicationMethod || "FULL_TABLE"
+      replicationMethod || "FULL_TABLE" /* FULL_TABLE */
     );
   }
   replicationMethod = (message) => {
@@ -2277,39 +2021,7 @@ var ITarget = class _ITarget {
             return translation;
           }
           return k;
-        }),
-        relationships: {
-          left: message.relationships.left.flatMap((l) => {
-            if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
-              const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
-              const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
-              if (from && to) {
-                return [{
-                  type: l.type,
-                  streamId: l.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          }),
-          right: message.relationships.right.flatMap((r) => {
-            if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
-              const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
-              const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
-              if (from && to) {
-                return [{
-                  type: r.type,
-                  streamId: r.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          })
-        }
+        })
       };
       await Bluebird3.map(this.schemaHooks, async (hook) => {
         const input = {
@@ -2342,9 +2054,6 @@ var ITarget = class _ITarget {
     try {
       logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
       await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
-      const configResolver = loadResolver();
-      await configResolver.markSyncComplete();
-      await configResolver.flushMetrics();
       return Promise.resolve();
     } catch (err) {
       logger.error(`Error while handling complete event.`);
@@ -2387,8 +2096,7 @@ var ITarget = class _ITarget {
     }
   };
   emitState = (state) => {
-    const configResolver = loadResolver();
-    return writeStateFile(configResolver, JSON.stringify(state));
+    return this.stateProvider.writeState(JSON.stringify(state));
   };
   uploadStreamsToWarehouse = async (streamsToUpload) => {
     try {
@@ -2460,15 +2168,15 @@ var safeTableName = (key) => {
 
 // src/targets/bigquery/service/record.ts
 import Decimal from "decimal.js";
-import moment5 from "moment";
+import dayjs5 from "dayjs";
 var validateDateRange = (record, schema) => {
   Object.keys(schema).forEach((key) => {
-    const { format: format3 } = schema[key];
-    if (format3 === "date-time") {
+    const { format: format6 } = schema[key];
+    if (format6 === "date-time") {
       if (record[key]) {
-        const formattedDate = moment5(record[key]).format(defaultDateTimeFormat);
-        const isNotTooMuchInTheFuture = moment5(record[key]).isBefore(moment5("9999-12-31 23:59:59.999999"));
-        const isNotTooMuchInThePast = moment5(record[key]).isAfter(moment5("0001-01-01 00:00:00"));
+        const formattedDate = dayjs5(record[key]).format(defaultDateTimeFormat);
+        const isNotTooMuchInTheFuture = dayjs5(record[key]).isBefore(dayjs5("9999-12-31 23:59:59.999999"));
+        const isNotTooMuchInThePast = dayjs5(record[key]).isAfter(dayjs5("0001-01-01 00:00:00"));
         if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
           record[key] = formattedDate;
         } else {
@@ -2523,7 +2231,7 @@ var convertNumberIntoDecimal = (record, schema) => {
 };
 
 // src/targets/bigquery/service/dbSync.ts
-import moment6 from "moment";
+import dayjs6 from "dayjs";
 
 // src/targets/bigquery/service/bigquery.ts
 import { BigQuery } from "@google-cloud/bigquery";
@@ -2604,13 +2312,13 @@ function v4(options, buf, offset) {
 var v4_default = v4;
 
 // src/targets/bigquery/service/cloudStorage.ts
-var uploadFileInBucket = async (streamId, gcsBuckerName, filePath) => {
+var uploadFileInBucket = async (streamId, gcsBuckerName, connectorId, filePath) => {
   try {
     const retryOptions = { autoRetry: true, maxRetries: 20 };
     let storage = new Storage({ retryOptions });
     const bucketRef = storage.bucket(gcsBuckerName);
     await bucketRef.get({ autoCreate: false });
-    const destinationFileName = `${streamId}-${v4_default()}.jsonnl`;
+    const destinationFileName = `${connectorId}/${streamId}-${v4_default()}.jsonnl`;
     await bucketRef.upload(filePath, {
       destination: destinationFileName
     });
@@ -2678,21 +2386,21 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
   safeColumnName = safeColumnName;
   getWarehouseTypeFromJSONSchema = (definition) => {
     const type = definition.type;
-    const format3 = definition.format;
+    const format6 = definition.format;
     let typeArr;
     if (type instanceof Array) {
       typeArr = type;
     } else {
       typeArr = [type];
     }
-    if (format3) {
-      switch (format3) {
+    if (format6) {
+      switch (format6) {
         case "date-time":
           return "TIMESTAMP";
         case "json":
           return "STRING";
         default:
-          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format3}`);
+          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format6}`);
       }
     } else if (typeArr.includes("number")) {
       return "NUMERIC";
@@ -2892,7 +2600,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
         await this.deleteStagingArea();
       }
       const schema = this.generateBigquerySchema();
-      const expirationTime = moment6().add("1", "day").valueOf().toString();
+      const expirationTime = dayjs6().add(1, "day").valueOf().toString();
       const createTableOptions = {
         schema,
         expirationTime
@@ -2915,6 +2623,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
       const file = await uploadFileInBucket(
         this.streamId,
         this.config.loading_deck_gcs_bucket_name,
+        this.config.connector_id,
         localFilePath
       );
       const metadata = {
@@ -3026,8 +2735,8 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
 
 // src/targets/bigquery/main.ts
 var BigQueryTarget = class extends ITarget {
-  constructor(config, resolver) {
-    super(config, resolver);
+  constructor(config, stateProvider) {
+    super(config, stateProvider);
     this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
   }
   static requiredConfigKeys = [
@@ -3050,6 +2759,75 @@ var BigQueryTarget = class extends ITarget {
   validateDateRange = validateDateRange;
   convertNumberIntoDecimal = convertNumberIntoDecimal;
 };
+
+// src/sdk/service/gcs.ts
+import { Storage as Storage2 } from "@google-cloud/storage";
+import { format as format4 } from "util";
+var getStorage = () => new Storage2({ retryOptions: { autoRetry: true, maxRetries: 20 } });
+var readObjectAsString = async (bucketName, objectPath) => {
+  try {
+    const storage = getStorage();
+    const bucketRef = storage.bucket(bucketName);
+    await bucketRef.get({ autoCreate: false });
+    const fileRef = bucketRef.file(objectPath);
+    const [exists] = await fileRef.exists();
+    if (!exists) {
+      throw new Error(`GCS object not found: gs://${bucketName}/${objectPath}`);
+    }
+    const [contents] = await fileRef.download();
+    return contents.toString("utf8");
+  } catch (err) {
+    throw new Error(format4(`error reading GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
+  }
+};
+var writeStringObject = async (bucketName, objectPath, contents) => {
+  try {
+    const storage = getStorage();
+    const bucketRef = storage.bucket(bucketName);
+    await bucketRef.get({ autoCreate: false });
+    const fileRef = bucketRef.file(objectPath);
+    await fileRef.save(contents, { contentType: "application/json" });
+    logger.info(`\u{1F5F3}  Uploaded object to gs://${bucketName}/${objectPath}`);
+  } catch (err) {
+    throw new Error(format4(`error writing GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
+  }
+};
+
+// src/state-providers/gcs/main.ts
+import { format as format5 } from "util";
+var GCSStateProvider = class {
+  bucketName;
+  connectorId;
+  constructor(connectorId, bucketName) {
+    this.connectorId = connectorId;
+    this.bucketName = bucketName;
+  }
+  getObjectPath() {
+    return `${this.connectorId}/state.json`;
+  }
+  async getState() {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
+      const raw = await readObjectAsString(this.bucketName, objectPath);
+      const parsed = JSON.parse(raw);
+      return { state: parsed };
+    } catch (err) {
+      logger.warn(format5(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
+      return { state: void 0 };
+    }
+  }
+  async writeState(state) {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
+      await writeStringObject(this.bucketName, objectPath, state);
+    } catch (err) {
+      logger.error(format5(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+      throw new Error(format5(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+};
 export {
   ALL_STREAM_ID_KEYWORD,
   API_CALLS_METRIC_NAME,
@@ -3059,6 +2837,7 @@ export {
   CounterMetric,
   DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME,
+  GCSStateProvider,
   ITarget,
   MissingFieldInSchemaError,
   MissingSchemaError,
@@ -3066,14 +2845,13 @@ export {
   ROWS_SYNCED_METRIC_NAME,
   RecordMessage,
   RenameColumnStore,
+  ReplicationMethod,
   ReplicationMethodMessage,
-  Resolver,
   SchemaMessage,
   SchemaValidationError,
   StateMessage,
   StateService,
   Stream,
-  StreamMetadata,
   StreamWarehouseSyncService,
   Tap,
   _greaterThan,
@@ -3081,7 +2859,6 @@ export {
   createTemporaryFileStream,
   extractStateForStream,
   finalizeStateProgressMarkers,
-  findRelatedRelationships,
   flattenSchema,
   getAllMetrics,
   getAxiosInstance,
@@ -3095,7 +2872,6 @@ export {
   haltAndCatchFire,
   incrementStreamState,
   loadJson,
-  loadResolver,
   loadSchema,
   logger,
   postFormDataApiCall,
@@ -3103,7 +2879,6 @@ export {
   postUrlEncodedApiCall,
   printMemoryFootprint,
   removeParasiteProperties,
-  safePath,
-  writeStateFile
+  safePath
 };
 //# sourceMappingURL=index.mjs.map