@whaly/connector-sdk 0.1.0 → 0.2.0

package/dist/index.mjs

@@ -2,243 +2,10 @@
 import { readFileSync } from "fs";
 
 // src/sdk/service/logger.ts
-import * as winston2 from "winston";
-
-// src/sdk/models/resolver.ts
-var Resolver = class {
-  constructor() {
-  }
-};
-
-// src/resolvers/local-file/services/files.ts
-import fs from "fs";
-var writeFile = (path, content) => {
-  return Promise.resolve(fs.writeFileSync(path, content));
-};
-var readAndParseJSONFile = (path) => {
-  if (!fs.existsSync(path)) {
-    throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
-  }
-  const raw = fs.readFileSync(path);
-  const parsed = JSON.parse(raw.toString());
-  return parsed;
-};
-
-// src/resolvers/local-file/hook/schemaHook.ts
-import { appendFile } from "fs-extra";
-var LocalSchemaHook = class {
-  async writeSchema(input) {
-    await appendFile("./schema.tmp", JSON.stringify(input) + "\n");
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-import winston from "winston";
-
-// src/sdk/service/metric.ts
-var metricStore = {};
-var ALL_STREAM_ID_KEYWORD = "all";
-var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-var API_CALLS_METRIC_NAME = "api_calls_count";
-var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-var getCounterMetrics = (name, streamIds) => {
-  return streamIds.map((streamId) => {
-    return getCounterMetric(name, streamId);
-  });
-};
-var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
-  if (!metricStore[streamId]) {
-    metricStore[streamId] = {};
-  }
-  const streamMetricStore = metricStore[streamId];
-  if (!streamMetricStore[name]) {
-    streamMetricStore[name] = new CounterMetric(name, streamId);
-  }
-  return streamMetricStore[name];
-};
-var getAllMetrics = () => {
-  return Object.keys(metricStore).flatMap((streamId) => {
-    const streamMetricStore = metricStore[streamId];
-    if (!streamMetricStore) {
-      return [];
-    }
-    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
-  });
-};
-var CounterMetric = class {
-  value = 0;
-  name;
-  streamId;
-  constructor(name, streamId) {
-    this.name = name;
-    this.streamId = streamId;
-  }
-  increment(inc = 1) {
-    this.value = this.value + inc;
-  }
-  getStreamId() {
-    return this.streamId;
-  }
-  getName() {
-    return this.name;
-  }
-  getValue() {
-    return this.value;
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
-var configPath = "./config.json";
-var targetConfigPath = "./target-config.json";
-var catalogPath = "./catalog.json";
-var statePath = "./state.json";
-var metricsPath = "./metrics.json";
-var LocalFilesResolver = class extends Resolver {
-  startVPNIfNeeded() {
-    return Promise.resolve();
-  }
-  checkIfCanSync() {
-    return Promise.resolve(true);
-  }
-  markSyncStarted() {
-    return Promise.resolve();
-  }
-  onError(errorType, errorText, errorDebugText) {
-    const path = `./error.json`;
-    return writeFile(path, JSON.stringify({
-      errorType,
-      errorText,
-      errorDebugText
-    }));
-  }
-  getSchemaHooks() {
-    const wlyHook = new LocalSchemaHook();
-    return [wlyHook];
-  }
-  writeSchema() {
-    return Promise.resolve();
-  }
-  constructor() {
-    super();
-  }
-  getConfig(command) {
-    logger.info(`\u{1F4C1} Using the LOCAL resolver`);
-    const config = readAndParseJSONFile(configPath);
-    const targetConfig = readAndParseJSONFile(targetConfigPath);
-    const destination = process.env.SHORE__DESTINATION;
-    if (!destination) {
-      throw new Error(`\u274C Env variable \`SHORE__DESTINATION\` is not set.
-            Did you forget to configure it?
-            
-            Possible values are: \`GOOGLE_BIGQUERY\`, \`SNOWFLAKE\``);
-    }
-    if (command === `READ`) {
-      if (destination === `GOOGLE_BIGQUERY`) {
-        const googleCredentials = process.env.GOOGLE_APPLICATION_CREDENTIALS;
-        if (!googleCredentials) {
-          throw new Error(`Env variable \`GOOGLE_APPLICATION_CREDENTIALS\` is not set. 
-                Did you forget to configure it?
-
-                This variable should link to a file containing the proper Google Service Account credentials.
-                `);
-        }
-      }
-      const catalog = readAndParseJSONFile(catalogPath);
-      const state = readAndParseJSONFile(statePath);
-      return Promise.resolve({
-        command,
-        destination,
-        config,
-        targetConfig,
-        catalog,
-        state
-      });
-    } else {
-      return Promise.resolve({
-        command,
-        destination,
-        config,
-        targetConfig
-      });
-    }
-  }
-  writeCatalog(catalog) {
-    return writeFile(catalogPath, JSON.stringify(catalog));
-  }
-  writeState(state) {
-    return writeFile(statePath, state);
-  }
-  markSyncFailed() {
-    logger.info(`\u{1F622} Sync has failed.`);
-    return Promise.resolve();
-  }
-  markSyncComplete() {
-    logger.info(`\u{1F389} Sync is complete.`);
-    return Promise.resolve();
-  }
-  markDiscoveryComplete() {
-    logger.info(`\u{1F389} Discovery is complete.`);
-    return Promise.resolve();
-  }
-  async flushMetrics() {
-    const allMetrics = getAllMetrics();
-    const metricsOuput = allMetrics.map((metric) => {
-      return {
-        streamId: metric.getStreamId(),
-        name: metric.getName(),
-        value: metric.getValue()
-      };
-    }).map((obj) => JSON.stringify(obj)).join(`
-`);
-    return writeFile(metricsPath, metricsOuput);
-  }
-  updateSourceValue(optionKey, optionValue) {
-    logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
-    const config = readAndParseJSONFile(configPath);
-    config[optionKey] = optionValue;
-    return writeFile(configPath, JSON.stringify(config));
-  }
-  getLogFormat() {
-    return combine(
-      timestamp(),
-      errors({ stack: true }),
-      splat(),
-      prettyPrint()
-    );
-  }
-};
-
-// src/sdk/service/resolver.ts
-function initResolver(resolverType) {
-  if (resolverType === "LOCAL") {
-    return new LocalFilesResolver();
-  } else {
-    throw new Error(`\u274C Unsupported configResolver: ${resolverType}`);
-  }
-}
-var loadResolver = () => {
-  const configResolver = process.env.SHORE__CONFIG_RESOLVER;
-  if (!configResolver) {
-    throw new Error(`Env variable \`SHORE__CONFIG_RESOLVER\` is not set. 
-        Did you forget to configure it?
-
-        Possible values are \`LOCAL\`, \`WHALY\`.
-        Please read the documentation to get the full description of the behavior of each.
-        `);
-  }
-  if (configResolver === "WHALY" || configResolver === "LOCAL") {
-    return initResolver(configResolver);
-  } else {
-    throw new Error(`Resolver: ${configResolver} is not supported.
-        Did you properly configure \`SHORE__CONFIG_RESOLVER\`?`);
-  }
-};
-
-// src/sdk/service/logger.ts
+import * as winston from "winston";
 var BATCH_INTERVAL_MS = 100;
 var transports2 = {
-  console: new winston2.transports.Console({
+  console: new winston.transports.Console({
     stderrLevels: [],
     level: "info"
   })
@@ -248,8 +15,14 @@ var getTransports = () => {
     transports2.console
   ];
 };
-var winstonLogger = winston2.createLogger({
-  format: loadResolver().getLogFormat(),
+var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
+var winstonLogger = winston.createLogger({
+  format: combine(
+    timestamp(),
+    errors({ stack: true }),
+    splat(),
+    prettyPrint()
+  ),
   transports: getTransports()
 });
 var isWinstonOpen = true;
@@ -278,34 +51,6 @@ var logger = {
 };
 
 // src/sdk/utils.ts
-var writeStateFile = (resolver, state) => {
-  logger.info(`\u{1F4DD} Writing the state file.`);
-  return resolver.writeState(state);
-};
-var writeCatalogFile = (resolver, catalog) => {
-  logger.info(`\u{1F4DD} Writing the catalog`);
-  return resolver.writeCatalog(catalog);
-};
-var loadShoreConfig = (resolver) => {
-  const command = process.env.SHORE__COMMAND;
-  if (!command) {
-    throw new Error(`Env variable \`SHORE__COMMAND\` is not set. 
-        Did you forget to configure it?
-
-        Possible values are: \`DISCOVER\`, \`READ\``);
-  }
-  if (command === `DISCOVER`) {
-    logger.info(`\u{1F50E} Mode: discovery
-        Shore will detect the available streams in the source and their configuration.`);
-  } else if (command === `READ`) {
-    logger.info(`\u{1F501} Mode: Synchronizing data
-        Shore will extract and push data from the source to the destination.`);
-  } else {
-    throw new Error(`Command: \`${command}\` is not supported.
-        Did you properly configured env variable \`SHORE__COMMAND\`?`);
-  }
-  return resolver.getConfig(command);
-};
 function readFile(fileName, filePath) {
   try {
     return readFileSync(filePath);
@@ -331,18 +76,11 @@ var loadSchema = (streamId) => {
   const schema = loadJson(`schemas/${streamId}.json`);
   return schema;
 };
-var checkRequiredConfigKeys = (args, requiredConfigKeys) => {
-  requiredConfigKeys.forEach((configKey) => {
-    if (!args[configKey]) {
-      throw new Error(`Config is missing required key: ${configKey}`);
-    }
-  });
-};
 
 // src/sdk/service/network.ts
 import axios from "axios";
 import { URLSearchParams } from "url";
-import * as fs2 from "fs";
+import * as fs from "fs";
 import * as qs from "qs";
 import axiosRetry, { isNetworkOrIdempotentRequestError } from "axios-retry";
 var shouldRetry = (error) => {
@@ -455,7 +193,7 @@ var getJSONApiCall = async (endpoint, config, privateLogging) => {
 };
 var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
   const instance = getAxiosInstance(config.retryCount);
-  const writer = fs2.createWriteStream(output);
+  const writer = fs.createWriteStream(output);
   const paramsSerializer = (params) => {
     return qs.stringify(params, { arrayFormat: "repeat" });
   };
@@ -549,6 +287,58 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
   }
 };
 
+// src/sdk/service/metric.ts
+var metricStore = {};
+var ALL_STREAM_ID_KEYWORD = "all";
+var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+var API_CALLS_METRIC_NAME = "api_calls_count";
+var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+var getCounterMetrics = (name, streamIds) => {
+  return streamIds.map((streamId) => {
+    return getCounterMetric(name, streamId);
+  });
+};
+var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
+  if (!metricStore[streamId]) {
+    metricStore[streamId] = {};
+  }
+  const streamMetricStore = metricStore[streamId];
+  if (!streamMetricStore[name]) {
+    streamMetricStore[name] = new CounterMetric(name, streamId);
+  }
+  return streamMetricStore[name];
+};
+var getAllMetrics = () => {
+  return Object.keys(metricStore).flatMap((streamId) => {
+    const streamMetricStore = metricStore[streamId];
+    if (!streamMetricStore) {
+      return [];
+    }
+    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
+  });
+};
+var CounterMetric = class {
+  value = 0;
+  name;
+  streamId;
+  constructor(name, streamId) {
+    this.name = name;
+    this.streamId = streamId;
+  }
+  increment(inc = 1) {
+    this.value = this.value + inc;
+  }
+  getStreamId() {
+    return this.streamId;
+  }
+  getName() {
+    return this.name;
+  }
+  getValue() {
+    return this.value;
+  }
+};
+
 // src/sdk/service/memory.ts
 var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
 var printMemoryFootprint = (prefix) => {
@@ -564,9 +354,6 @@ var printMemoryFootprint = (prefix) => {
 
 // src/sdk/service/exit.ts
 async function gracefulExit(exitCode) {
-  if (exitCode !== 0) {
-    await loadResolver().markSyncFailed();
-  }
   const loggerFinish = new Promise((resolve, reject) => {
     logger.closeWinston(resolve);
   });
@@ -586,207 +373,15 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
     errorText,
     errorDebugText
   );
-  const resolver = loadResolver();
-  await resolver.onError(errorType, errorText, errorDebugText);
   await gracefulExit(0);
 };
 
-// src/sdk/service/relationship.ts
-var findRelatedRelationships = (streamId, allRels) => {
-  return allRels.reduce((acc, rel) => {
-    const base = {
-      type: rel.type,
-      from: rel.from,
-      to: rel.to
-    };
-    if (rel.left === streamId) {
-      acc.right.push({
-        streamId: rel.right,
-        ...base
-      });
-    }
-    if (rel.right === streamId) {
-      acc.left.push({
-        streamId: rel.left,
-        ...base
-      });
-    }
-    return acc;
-  }, { left: [], right: [] });
-};
-
-// src/sdk/models/metadata.ts
-var MetadataMap = class extends Map {
-  areEquals(array1, array2) {
-    return array1.length === array2.length && array1.every(function(value, index) {
-      return value === array2[index];
-    });
-  }
-  findExistingEntry(key) {
-    return Array.from(super.entries()).find((entry) => {
-      if (this.areEquals(entry[0], key)) {
-        return true;
-      }
-    });
-  }
-  get(key) {
-    const existingEntry = this.findExistingEntry(key);
-    return existingEntry?.[1];
-  }
-  set(key, value) {
-    const existingKey = this.findExistingEntry(key);
-    if (existingKey) {
-      return super.set(existingKey?.[0], value);
-    } else {
-      return super.set(key, value);
-    }
-  }
-};
-var StreamMetadata = class {
-  metadataByBreadcrumb;
-  constructor() {
-    this.metadataByBreadcrumb = new MetadataMap();
-  }
-  fromCatalogStreamMetadata(metadataArr) {
-    metadataArr.forEach((metadata) => {
-      this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
-    });
-    return this;
-  }
-  toString() {
-    return this.metadataByBreadcrumb;
-  }
-  get(breadcrumb) {
-    return this.metadataByBreadcrumb.get(breadcrumb);
-  }
-  write(breadcrumb, key, value) {
-    if (value === void 0) {
-      throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
-            
-            Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
-    }
-    if (!this.metadataByBreadcrumb.get(breadcrumb)) {
-      this.metadataByBreadcrumb.set(breadcrumb, {});
-    }
-    const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
-    const newValue = {};
-    newValue[key] = value;
-    this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
-    return this;
-  }
-  getRootBreadcrumbMetadata() {
-    const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
-    if (!rootBreadcrumbMetadata) {
-      throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level. 
-            
-            Is your catalog properly generated?`);
-    }
-    return rootBreadcrumbMetadata;
-  }
-  isStreamSelected() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (rootBreadcrumbMetadata.selected === false) {
-      return false;
-    }
-    if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
-      return true;
-    }
-    return false;
-  }
-  getKeyProperties() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["table-key-properties"]) {
-      return [];
-    }
-    return rootBreadcrumbMetadata["table-key-properties"];
-  }
-  getReplicationKey() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-key"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-key"];
-  }
-  getReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-method"];
-  }
-  getForcedReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["forced-replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["forced-replication-method"];
-  }
-  toList() {
-    return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
-      const metadata = this.metadataByBreadcrumb.get(breadcrumb);
-      if (!metadata) {
-        throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
-      }
-      return {
-        breadcrumb,
-        metadata
-      };
-    });
-  }
-};
-
-// src/sdk/models/catalog.ts
-import * as _ from "lodash";
-var Catalog = class _Catalog {
-  streams;
-  static fromFile(fileName) {
-    const catalogFile = loadJson(fileName);
-    const catalog = new _Catalog();
-    catalog.streams = catalogFile.streams;
-    return catalog;
-  }
-  static fromCatalogFile(catalogFile) {
-    const catalog = new _Catalog();
-    catalog.streams = catalogFile.streams;
-    return catalog;
-  }
-  getSelectedStreams() {
-    if (!this.streams) {
-      return [];
-    }
-    const directSelectedStreams = this.streams.filter((stream) => {
-      const metadata = new StreamMetadata();
-      metadata.fromCatalogStreamMetadata(stream.metadata);
-      return metadata.isStreamSelected();
-    });
-    logger.debug(`Directly selected streams: ${directSelectedStreams.map((stream) => stream.stream).join(",")}`);
-    const getIndirectSelectedStreams = (stream, ancestorsIds) => {
-      const currAncestorsIds = ancestorsIds || [];
-      const rootBreadcrumbMetadata = stream.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-      const parentStreamId = rootBreadcrumbMetadata?.metadata?.["whaly-parent-stream"];
-      if (parentStreamId) {
-        const parentStream = this.streams?.find((stream2) => stream2.stream === parentStreamId);
-        if (parentStream) {
-          currAncestorsIds.push(parentStreamId);
-          return getIndirectSelectedStreams(parentStream, currAncestorsIds);
-        }
-      }
-      return currAncestorsIds;
-    };
-    const indirectSelectedStreamIds = _.uniq(
-      directSelectedStreams.flatMap((stream) => {
-        return getIndirectSelectedStreams(stream);
-      })
-    );
-    logger.debug(`Indirectly selected streams: ${indirectSelectedStreamIds.join(",")}`);
-    const indirectSelectedStreams = indirectSelectedStreamIds.map((streamId) => {
-      if (!directSelectedStreams.find((stream) => stream.stream === streamId)) {
-        return this.streams?.find((stream) => stream.stream === streamId);
-      }
-    }).filter((stream) => !!stream);
-    return directSelectedStreams.concat(indirectSelectedStreams);
-  }
-};
+// src/sdk/models/replication.ts
+var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
+  ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
+  ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
+  return ReplicationMethod2;
+})(ReplicationMethod || {});
 
 // src/sdk/models/messages.ts
 var RecordMessage = class {
@@ -846,7 +441,6 @@ var SchemaMessage = class {
   displayLabel;
   description;
   propertiesMetadata;
-  relationships;
   constructor(opts) {
     const {
       stream,
@@ -855,7 +449,6 @@ var SchemaMessage = class {
       bookmarkProperties,
       displayLabel,
       description,
-      relationships,
       propertiesMetadata
     } = opts;
     this.stream = stream;
@@ -864,7 +457,6 @@ var SchemaMessage = class {
     this.bookmarkProperties = bookmarkProperties;
     this.displayLabel = displayLabel;
     this.description = description;
-    this.relationships = relationships;
     this.propertiesMetadata = propertiesMetadata;
   }
   toString() {
@@ -874,8 +466,7 @@ var SchemaMessage = class {
       schema: this.schema,
       key_properties: this.keyProperties,
       label: this.displayLabel,
-      description: this.description,
-      relationships: this.relationships
+      description: this.description
     };
     if (this.bookmarkProperties) {
       result["bookmark_properties"] = this.bookmarkProperties;
@@ -885,27 +476,27 @@ var SchemaMessage = class {
 };
 
 // src/sdk/models/state.ts
-import moment from "moment";
+import dayjs from "dayjs";
 
 // src/sdk/constants/date.ts
 var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
 
 // src/sdk/models/state.ts
-import { format } from "util";
+import { format as format2 } from "util";
 var PROGRESS_MARKERS = "progressMarkers";
 var PROGRESS_MARKER_NOTE = "Note";
 var SIGNPOST_MARKER = "replicationKeySignpost";
 var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
   let newRkValue = latestRecord[replicationKey];
   if (!newRkValue) {
-    throw new Error(format(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
+    throw new Error(format2(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
   }
-  let newRkValueMoment = moment(newRkValue);
+  let newRkValueMoment = dayjs(newRkValue);
   let prevRkValue;
   let prevRkValueMoment;
   if (isSorted) {
     prevRkValue = state["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue);
+    prevRkValueMoment = dayjs(prevRkValue);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       throw new Error(
         `Unsorted data detected in stream. Latest value '${newRkValue}' is
@@ -920,24 +511,24 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
       state[PROGRESS_MARKERS] = marker;
     }
     prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue || state["replicationKeyValue"]);
+    prevRkValueMoment = dayjs(prevRkValue || state["replicationKeyValue"]);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       newRkValueMoment = prevRkValueMoment;
       newRkValue = prevRkValue;
     }
   }
   const signpostMarker = state[SIGNPOST_MARKER];
-  const replicationKeySignpostMoment = moment(signpostMarker);
+  const replicationKeySignpostMoment = dayjs(signpostMarker);
   if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
     newRkValueMoment = replicationKeySignpostMoment;
     newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
   }
   if (isSorted === false) {
     state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
-    state[PROGRESS_MARKERS]["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state[PROGRESS_MARKERS]["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   } else {
     state["replicationKey"] = replicationKey;
-    state["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   }
   return state;
 };
@@ -959,7 +550,7 @@ var finalizeStateProgressMarkers = (state) => {
   return state;
 };
 var _greaterThan = (aValue, bValue) => {
-  return moment(aValue).isAfter(moment(bValue));
+  return dayjs(aValue).isAfter(dayjs(bValue));
 };
 var extractStateForStream = (state, streamId) => {
   if (!state.bookmarks) {
@@ -982,26 +573,16 @@ var StateService = class {
     }
     return this.instance;
   }
-  // To be deprecated
-  setBookmark(streamId, ts) {
-    this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
-  }
-  setBookmarkV2(streamId, streamState) {
+  setBookmark(streamId, streamState) {
     this.bookmarks[streamId] = streamState;
   }
-  setBookmarkSignpostV2(streamId, signpostValue) {
+  setBookmarkSignpost(streamId, signpostValue) {
     if (!this.bookmarks[streamId]) {
       this.bookmarks[streamId] = {};
     }
     this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
   }
   getBookmark(streamId) {
-    if (!this.bookmarks[streamId]) {
-      return moment(0);
-    }
-    return moment(this.bookmarks[streamId]);
-  }
-  getBookmarkV2(streamId) {
     if (this.bookmarks[streamId]) {
       return this.bookmarks[streamId];
     } else {
@@ -1041,7 +622,7 @@ import axios2 from "axios";
 import * as qs2 from "qs";
 
 // src/sdk/models/tap/stream.ts
-import moment2 from "moment";
+import dayjs2 from "dayjs";
 
 // src/sdk/helpers/typing.ts
 var isDatetimeType = (typeDef) => {
@@ -1064,186 +645,56 @@ var isDatetimeType = (typeDef) => {
 };
 
 // src/sdk/models/tap/stream.ts
-import * as _2 from "lodash";
-import { Promise as BPromise2 } from "bluebird";
+import cloneDeep2 from "lodash/cloneDeep.js";
+import Bluebird2 from "bluebird";
 
 // src/sdk/models/tap/tap.ts
-import { Promise as BPromise } from "bluebird";
-import { DepGraph } from "dependency-graph";
+import cloneDeep from "lodash/cloneDeep.js";
+import Bluebird from "bluebird";
 var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
 var Tap = class {
-  _target;
-  _config;
-  _streams;
-  _concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
-  constructor(target, config) {
-    this._streams = [];
-    this._target = target;
-    this._config = config;
-  }
-  discover = async (existingCatalog) => {
-    logger.debug("Loading schemas");
-    const streamDepGraph = new DepGraph();
-    const buildDependencyGraph = (streams2, parentId) => {
-      streams2.forEach((s) => {
-        streamDepGraph.addNode(s.streamId, s);
-        if (parentId) {
-          streamDepGraph.addDependency(s.streamId, parentId);
-        }
-        if (s.children) {
-          return buildDependencyGraph(s.children, s.streamId);
-        }
-      });
-    };
-    buildDependencyGraph(this._streams);
-    const discoveredStreamCatalogEntries = await BPromise.map(
-      streamDepGraph.overallOrder(),
-      async (streamId) => {
-        logger.info(`\u2728 Loading schema for ${streamId}`);
-        const stream = streamDepGraph.getNodeData(streamId);
-        const schema = await stream.getSchema();
-        const parentStreamIds = streamDepGraph.directDependenciesOf(streamId);
-        if (parentStreamIds.length > 1) {
-          throw new Error(`StreamId=${streamId} directly depends on more than 1 stream, e.g. ${parentStreamIds.join(",")} which is not supported!`);
-        }
-        const parentStreamId = parentStreamIds?.[0];
-        const { metadata } = await this.generateMetadataFromStream(stream, parentStreamId);
-        return {
-          schema,
-          metadata,
-          stream: stream.streamId,
-          tap_stream_id: stream.streamId
-        };
-      },
-      { concurrency: this._concurrency }
-    );
-    const existingCatalogStreamIds = (existingCatalog?.streams || []).map((entry) => {
-      return entry.stream;
-    });
-    const newlyDiscoveredStreamCatalogEntries = discoveredStreamCatalogEntries.filter((entry) => {
-      if (existingCatalogStreamIds.includes(entry.stream)) {
-        return false;
-      } else {
-        return true;
-      }
-    });
-    logger.debug(`Newly discovered streams: %j`, newlyDiscoveredStreamCatalogEntries);
-    const allDiscoveredStreamIds = discoveredStreamCatalogEntries.map((entry) => {
-      return entry.stream;
-    });
-    const existingCatalogMinusDeletedStreams = (existingCatalog?.streams || []).filter((entry) => {
-      if (!allDiscoveredStreamIds.includes(entry.stream)) {
-        logger.debug(`Stream %s was deleted, removing it from the catalog`, entry.stream);
-        return false;
-      } else {
-        return true;
-      }
-    });
-    const updatedExistingCatalog = existingCatalogMinusDeletedStreams.map((existingCatalogEntry) => {
-      const matchingDiscoveredCatalogEntry = discoveredStreamCatalogEntries.find((entry) => entry.stream === existingCatalogEntry.stream);
-      if (matchingDiscoveredCatalogEntry) {
-        const existingRootMetadata = existingCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-        const discoveredRootMetadata = matchingDiscoveredCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-        if (existingRootMetadata) {
-          const parentStreamVal = discoveredRootMetadata?.metadata["whaly-parent-stream"];
-          if (parentStreamVal !== void 0) {
-            existingRootMetadata.metadata["whaly-parent-stream"] = parentStreamVal;
-          }
-          const forcedReplicationMethodVal = discoveredRootMetadata?.metadata["forced-replication-method"];
-          if (forcedReplicationMethodVal !== void 0) {
-            existingRootMetadata.metadata["forced-replication-method"] = forcedReplicationMethodVal;
-          }
-          const forcedReplicationKeyVal = discoveredRootMetadata?.metadata["forced-replication-key"];
-          if (forcedReplicationKeyVal !== void 0) {
-            existingRootMetadata.metadata["forced-replication-key"] = forcedReplicationKeyVal;
-          }
-          const validReplicationKeysVal = discoveredRootMetadata?.metadata["valid-replication-keys"];
-          if (validReplicationKeysVal !== void 0) {
-            existingRootMetadata.metadata["valid-replication-keys"] = validReplicationKeysVal;
-          }
-          const rowCountVal = discoveredRootMetadata?.metadata["row-count"];
-          if (rowCountVal !== void 0) {
-            existingRootMetadata.metadata["row-count"] = rowCountVal;
-          }
-        }
-      }
-      return existingCatalogEntry;
-    });
-    const streams = updatedExistingCatalog.concat(newlyDiscoveredStreamCatalogEntries);
-    return { streams };
-  };
-  sync = async (catalog) => {
-    await this.initiateSync(catalog);
-    logger.debug(`[TAP] All streams have finished syncing. We send the [complete] signal to the target.`);
-    await this._target.complete();
-    return Promise.resolve();
-  };
-  async generateMetadataFromStream(stream, parentStreamId) {
-    const metadata = new StreamMetadata();
-    metadata.write([], "table-key-properties", stream.primaryKey);
-    const defaultReplicationConfig = stream.defaultReplicationConfig();
-    metadata.write([], "replication-method", defaultReplicationConfig.replicationMethod);
-    const validReplicationkeys = await stream.getValidReplicationKeys();
-    metadata.write([], "valid-replication-keys", validReplicationkeys);
-    if (defaultReplicationConfig.isForced) {
-      metadata.write([], "forced-replication-method", defaultReplicationConfig.replicationMethod);
-      if (stream.forcedReplicationKey) {
-        metadata.write([], "forced-replication-key", stream.forcedReplicationKey);
-      }
-    }
-    const rowCount = await stream.getRowCount();
-    if (rowCount !== void 0) {
-      metadata.write([], "row-count", rowCount);
-    }
-    if (stream.isSilent === true) {
-      metadata.write([], "whaly-is-hidden", stream.isSilent);
-    }
-    if (stream.isSilent === false) {
-      metadata.write([], "selected-by-default", stream.selectedByDefault);
-    }
-    if (stream.displayLabel) {
-      metadata.write([], "whaly-display-label", stream.displayLabel);
-    }
-    if (stream.description) {
-      metadata.write([], "whaly-description", stream.description);
-    }
-    if (parentStreamId !== void 0) {
-      metadata.write([], "whaly-parent-stream", parentStreamId);
-    }
-    return { metadata: metadata.toList() };
+  target;
+  config;
+  streams;
+  concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
+  stateProvider;
+  tapState;
+  constructor(target, config, stateProvider) {
+    this.streams = [];
+    this.target = target;
+    this.config = config;
+    this.stateProvider = stateProvider;
+    this.tapState = { bookmarks: {} };
   }
-  async initiateSync(catalog) {
+  sync = async (options) => {
+    const initialState = await this.stateProvider.getState();
+    if (initialState.state && initialState.state.bookmarks) {
+      this.tapState = cloneDeep(initialState.state);
+    }
+    await this.init();
     logger.info(`\u{1F680} Start syncing`);
     const state = StateService.getInstance().get();
     logger.info(`\u{1F4CD} Received state: %j`, state);
-    logger.debug(`\u{1F4DA} Received catalog: %j`, catalog);
-    const selectedStreams = catalog.getSelectedStreams();
-    if (selectedStreams.length === 0) {
-      logger.info(`\u{1F631} There is no stream to sync. 
-            
-            Did you update the \`catalog.json\` to add \`"selected": true\` in the table metadata associated with the root breadcrumb (e.g. \`[]\`)`);
-      return;
-    }
-    const selectedStreamIds = selectedStreams.map((stream) => stream.stream);
-    logger.info(`\u{1F973} Syncing selected streams: %s`, selectedStreamIds.join(","));
-    await BPromise.map(
-      this._streams,
-      (stream) => {
-        if (selectedStreamIds.includes(stream.streamId)) {
-          const selectedStream = selectedStreams.find(((selectedStream2) => selectedStream2.stream === stream.streamId));
-          const rootMetadataEntry = selectedStream?.metadata.find((streamMdt) => streamMdt.breadcrumb.length === 0);
-          if (!rootMetadataEntry) {
-            throw new Error(`Can't find a catalog root metadata entry for StreamId=${stream.streamId}`);
-          }
-          stream.setRootMetadataEntry(rootMetadataEntry.metadata);
-          return stream.sync();
-        }
-      },
-      {
-        concurrency: this._concurrency
-      }
-    );
-  }
+    const envInclude = process.env.TAP_STREAMS ? process.env.TAP_STREAMS.split(",").map((s) => s.trim()).filter((s) => s.length > 0) : void 0;
+    const include = options?.include && options.include.length > 0 ? options.include : envInclude;
+    const exclude = new Set((options?.exclude || []).map((s) => s.trim()));
+    let streamsToRun = this.streams.filter((s) => !s.isSilent);
+    if (include && include.length > 0) {
+      const includeSet = new Set(include);
+      streamsToRun = streamsToRun.filter((s) => includeSet.has(s.streamId));
+    }
+    if (exclude.size > 0) {
+      streamsToRun = streamsToRun.filter((s) => !exclude.has(s.streamId));
+    }
+    const selectedStreamIds = streamsToRun.map((s) => s.streamId);
+    logger.info(`\u{1F973} Syncing streams: %s`, selectedStreamIds.join(","));
+    await Bluebird.map(streamsToRun, (stream) => {
+      return stream.sync();
+    }, { concurrency: this.concurrency });
+    logger.debug(`[TAP] All streams have finished syncing. We send the [complete] signal to the target.`);
+    await this.target.complete();
+    return Promise.resolve();
+  };
   // Used to clean up any ressource before exit
   end() {
     return Promise.resolve();
@@ -1251,19 +702,13 @@ var Tap = class {
 };
 
 // src/sdk/models/tap/stream.ts
-import { format as format2 } from "util";
-var StreamV2 = class {
+import { format as format3 } from "util";
+var Stream = class {
   // Runtime values, can't be overriden
-  _config;
-  _tapName = "default";
-  _tapState;
-  _target;
-  // Contains the root metadata entry of the catalog stream configuration
-  // contains some end user schema configuration
-  rootMetadataEntry;
-  // Compile time values, can be overriden
-  // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
-  forcedReplicationMethod = void 0;
+  config;
+  tapState;
+  target;
+  replicationMethod = void 0;
   selectedByDefault = true;
   STATE_MSG_FREQUENCY = 10;
   streamId = "default";
@@ -1271,10 +716,9 @@ var StreamV2 = class {
   displayLabel = void 0;
   description = void 0;
   primaryKey = [];
-  forcedReplicationKey = void 0;
+  replicationKey = void 0;
   // When set, use the state from another stream for this stream
   useStateFromStreamId = void 0;
-  relationships = [];
   children = [];
   // If true, it means that the records in this stream are sorted by replicationKey.
   // If false, then the records are coming in an unsorted manner.
@@ -1286,11 +730,10 @@ var StreamV2 = class {
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
-  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
-    this._tapName = tapName;
-    this._config = config;
-    this._tapState = _2.cloneDeep(state);
-    this._target = target;
+  constructor(config, tapState, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    this.config = config;
+    this.tapState = cloneDeep2(tapState);
+    this.target = target;
     this.childConcurrency = childConcurrency;
     this.rowsSyncedMetricsConf = {
       isEnabled: () => !this.isSilent,
@@ -1301,9 +744,7 @@ var StreamV2 = class {
       getStreamIds: () => [this.streamId]
     };
   }
-  setRootMetadataEntry = (rootMetadataEntry) => {
-    this.rootMetadataEntry = rootMetadataEntry;
-  };
+  // No root metadata: replication config is derived from stream defaults only
   /**
    * Sync this stream
    * Called only for streams with no parents, aka. "root tap streams"
@@ -1323,7 +764,7 @@ var StreamV2 = class {
       return this.flush();
     } catch (err) {
       if (err instanceof Error) {
-        throw new Error(format2(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
+        throw new Error(format3(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
       }
       throw err;
     }
@@ -1334,7 +775,7 @@ var StreamV2 = class {
       `\u{1F6BD} Flushing ${replicationMethod} sync of stream '${this.streamId}' + all of its children...`
     );
     await this._flush();
-    await BPromise2.map(this.children, (child) => {
+    await Bluebird2.map(this.children, (child) => {
       return child.flush();
     }, { concurrency: 3 });
   };
@@ -1344,7 +785,7 @@ var StreamV2 = class {
    */
   _writeStateMessage = () => {
     const message = new StateMessage(StateService.getInstance().get());
-    return this._target.state(message);
+    return this.target.state(message);
   };
   /**
    * Write out a ACTIVATE_VERSION message.
@@ -1352,10 +793,10 @@ var StreamV2 = class {
   _writeReplicationMethodMessage = async () => {
     const method = this.configuredReplicationConfig().replicationMethod;
     const message = new ReplicationMethodMessage(this.streamId, method);
-    await BPromise2.map(this.children, async (c) => {
+    await Bluebird2.map(this.children, async (c) => {
       return c._writeReplicationMethodMessage();
     }, { concurrency: 3 });
-    await this._target.replicationMethod(message);
+    await this.target.replicationMethod(message);
   };
   /**
    * Write out a SCHEMA message with the stream schema.
@@ -1364,23 +805,21 @@ var StreamV2 = class {
     if (!this.isSilent) {
       const schema = await this.getSchema();
       if (schema !== void 0) {
-        const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
         const replicationConfig = this.configuredReplicationConfig();
         const message = new SchemaMessage({
           keyProperties: this.primaryKey,
           stream: this.streamId,
           schema: schema.jsonSchema,
-          relationships: relatedRels,
           ...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
           ...this.displayLabel ? { displayLabel: this.displayLabel } : {},
           ...this.description ? { description: this.description } : {},
           ...schema.propertiesMetadata ? { propertiesMetadata: schema.propertiesMetadata } : {}
         });
-        await this._target.schema(message);
+        await this.target.schema(message);
       }
     }
     if (skipChildrenSchema !== true) {
-      await BPromise2.map(this.children, async (child) => {
+      await Bluebird2.map(this.children, async (child) => {
         await child._writeSchemaMessage();
       }, { concurrency: 3 });
     }
@@ -1390,7 +829,7 @@ var StreamV2 = class {
     const signpostMoment = await this.getReplicationKeySignpost();
     const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
     if (signpostValue) {
-      StateService.getInstance().setBookmarkSignpostV2(this.streamId, signpostValue);
+      StateService.getInstance().setBookmarkSignpost(this.streamId, signpostValue);
     }
   };
   /**
@@ -1406,7 +845,7 @@ var StreamV2 = class {
         if (this.useStateFromStreamId) {
           return;
         }
-        if (this.forcedReplicationMethod && !this.forcedReplicationKey) {
+        if (this.replicationMethod && !this.replicationKey) {
           return;
         }
         if (!replicationConfig.replicationKey) {
@@ -1416,14 +855,14 @@ var StreamV2 = class {
           );
         }
         const stateServiceInst = StateService.getInstance();
-        const state = stateServiceInst.getBookmarkV2(this.streamId);
+        const state = stateServiceInst.getBookmark(this.streamId);
         const newState = incrementStreamState(
           state,
           replicationConfig.replicationKey,
           latestRecord,
           this.isSorted
         );
-        StateService.getInstance().setBookmarkV2(this.streamId, newState);
+        StateService.getInstance().setBookmark(this.streamId, newState);
       }
     }
   };
@@ -1455,27 +894,27 @@ var StreamV2 = class {
    **/
   getStartingTimestamp() {
     const streamIdToReadFrom = this.useStateFromStreamId ? this.useStateFromStreamId : this.streamId;
-    const state = extractStateForStream(this._tapState, streamIdToReadFrom);
-    const startDate = this._config.start_date;
+    const state = extractStateForStream(this.tapState, streamIdToReadFrom);
+    const startDate = this.config.start_date;
     const replicationConfig = this.configuredReplicationConfig();
     logger.debug(`${this.streamId}: state.replicationKeyValue: ${state.replicationKeyValue}, 
         state.replicationKey: ${state.replicationKey},
         this.replicationKey: ${replicationConfig.replicationKey}`);
     if (state.replicationKeyValue) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
-      return moment2(state.replicationKeyValue);
+      return dayjs2(state.replicationKeyValue);
     } else if (startDate) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
-      return moment2(startDate);
+      return dayjs2(startDate);
     } else {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
-      return moment2(0);
+      return dayjs2(0);
     }
   }
   /**
    * Return the max allowable bookmark value for this stream's replication key.
    * 
-   * For timestamp-based replication keys, this defaults to `moment()`. For
+   * For timestamp-based replication keys, this defaults to `dayjs()`. For
    * non-timestamp replication keys, default to `undefined`.
    * 
    * Override this value to prevent bookmarks from being advanced in cases where we
@@ -1483,7 +922,7 @@ var StreamV2 = class {
    */
   getReplicationKeySignpost = async () => {
     if (await this.isTimestampReplicationKey()) {
-      return moment2();
+      return dayjs2();
     }
     return void 0;
   };
@@ -1491,22 +930,22 @@ var StreamV2 = class {
    * Return the default replication method for the stream that will be used to write the catalog.
    */
   defaultReplicationConfig = () => {
-    if (this.forcedReplicationMethod) {
+    if (this.replicationMethod) {
       return {
-        replicationMethod: this.forcedReplicationMethod,
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: this.replicationMethod,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
-    if (this.forcedReplicationKey || this.useStateFromStreamId) {
+    if (this.replicationKey || this.useStateFromStreamId) {
       return {
-        replicationMethod: "INCREMENTAL",
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: "INCREMENTAL" /* INCREMENTAL */,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
     return {
-      replicationMethod: "FULL_TABLE",
+      replicationMethod: "FULL_TABLE" /* FULL_TABLE */,
       replicationKey: void 0,
       isForced: false
     };
@@ -1515,16 +954,10 @@ var StreamV2 = class {
    * Return the configured replication method that will be used for the sync.
    */
   configuredReplicationConfig = () => {
-    const defaultReplicationMethod = this.defaultReplicationConfig();
-    if (defaultReplicationMethod.isForced) {
-      return {
-        replicationMethod: defaultReplicationMethod.replicationMethod,
-        replicationKey: defaultReplicationMethod.replicationKey
-      };
-    }
+    const defaults = this.defaultReplicationConfig();
     return {
-      replicationMethod: this.rootMetadataEntry?.["replication-method"] || defaultReplicationMethod.replicationMethod,
-      replicationKey: this.rootMetadataEntry?.["replication-key"] || defaultReplicationMethod.replicationKey
+      replicationMethod: defaults.replicationMethod,
+      replicationKey: defaults.replicationKey
     };
   };
   // Private sync methods:
@@ -1554,9 +987,9 @@ var StreamV2 = class {
         this.streamId
       );
       if (!this.isSilent) {
-        await this._target.record(recordMessage);
+        await this.target.record(recordMessage);
       }
-      await BPromise2.map(
+      await Bluebird2.map(
         this.children,
         async (child) => {
           await child.asyncInit(row);
@@ -1573,9 +1006,9 @@ var StreamV2 = class {
       rowsSent += 1;
     }
     const stateServiceInst = StateService.getInstance();
-    const state = stateServiceInst.getBookmarkV2(this.streamId);
+    const state = stateServiceInst.getBookmark(this.streamId);
     const newState = finalizeStateProgressMarkers(state);
-    stateServiceInst.setBookmarkV2(this.streamId, newState);
+    stateServiceInst.setBookmark(this.streamId, newState);
     if (!parent) {
       logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
     } else {
@@ -1600,7 +1033,7 @@ var StreamV2 = class {
   getSchema() {
     logger.debug(`stream: ${this.streamId} - getSchema() is called`);
     if (this.schemaPath) {
-      const schema = loadJson(`schemas/${this._tapName}/${this.schemaPath}`);
+      const schema = loadJson(`schemas/${this.schemaPath}`);
       return Promise.resolve({ jsonSchema: schema });
     } else {
       if (this.isSilent) {
@@ -1637,7 +1070,7 @@ var StreamV2 = class {
 };
 
 // src/sdk/helpers/qs-logger.ts
-import * as _3 from "lodash";
+import isPlainObject from "lodash/isPlainObject.js";
 var MAX_QS_KEYS = 10;
 var MAX_QS_STRING_LENGTH = 200;
 var MAX_QS_ARRAY_ITEMS = 5;
@@ -1658,7 +1091,7 @@ function truncateQueryParams(value, depth = 0) {
     }
     return items;
   }
-  if (_3.isPlainObject(value)) {
+  if (isPlainObject(value)) {
     const keys = Object.keys(value);
     const result = {};
     for (const key of keys.slice(0, MAX_QS_KEYS)) {
@@ -1681,11 +1114,12 @@ function getTruncatedParamsForLog(params) {
 }
 
 // src/sdk/models/tap/restStream.ts
-import * as _4 from "lodash";
-var RESTStreamV2 = class extends StreamV2 {
+import cloneDeep3 from "lodash/cloneDeep.js";
+import isEqual from "lodash/isEqual.js";
+var RESTStream = class extends Stream {
   axiosInstance;
-  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
-    super(tapName, config, state, target, childConcurrency);
+  constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    super(config, state, target, childConcurrency);
     this.apiCallsMetricsConf = {
       isEnabled: () => !this.isSilent,
       getStreamIds: () => [this.streamId]
@@ -1701,14 +1135,25 @@ var RESTStreamV2 = class extends StreamV2 {
    */
   _prepareRequest = async (nextPageToken, parent) => {
     const method = this.httpMethod;
-    const url = this.getUrl(parent);
+    const url = this.getNextUrl(nextPageToken, parent);
     if (!url) {
       throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
     }
     const authenticator = this.authenticator;
+    let finalUrl = url;
+    let urlQueryParams = {};
+    const queryIndex = url.indexOf("?");
+    if (queryIndex !== -1) {
+      finalUrl = url.substring(0, queryIndex);
+      const rawQueryString = url.substring(queryIndex + 1);
+      urlQueryParams = qs2.parse(rawQueryString);
+    }
     const params = {
       ...this.getNextUrlParams(nextPageToken, parent),
-      ...await authenticator?.getAuthQS(this._config)
+      // Query params embedded in URL should override computed pagination params
+      // but should not override auth params which are appended last
+      ...urlQueryParams,
+      ...await authenticator?.getAuthQS(this.config)
     };
     const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
     let headers = this.getCustomHTTPHeaders(parent);
@@ -1716,7 +1161,7 @@ var RESTStreamV2 = class extends StreamV2 {
       headers = {};
     }
     if (authenticator) {
-      const authHeaders = await authenticator.getAuthHeaders(this._config);
+      const authHeaders = await authenticator.getAuthHeaders(this.config);
       headers = {
         ...headers,
         ...authHeaders
@@ -1726,7 +1171,7 @@ var RESTStreamV2 = class extends StreamV2 {
       return qs2.stringify(params2, { arrayFormat: "repeat" });
     };
     const request = {
-      url,
+      url: finalUrl,
       method,
       data: requestBody,
       params,
@@ -1799,9 +1244,9 @@ var RESTStreamV2 = class extends StreamV2 {
       for await (const row of rows) {
         yield row;
       }
-      const previousToken = _4.cloneDeep(nextPageToken);
+      const previousToken = cloneDeep3(nextPageToken);
       nextPageToken = this.getNextPageToken(resp, previousToken);
-      if (nextPageToken && _4.isEqual(nextPageToken, previousToken)) {
+      if (nextPageToken && isEqual(nextPageToken, previousToken)) {
         throw new Error(
           `${this.streamId} - Loop detected in pagination. "
                     Pagination token: \`${JSON.stringify(nextPageToken)}\` is identical to prior token \`${JSON.stringify(previousToken)}\`.`
@@ -1878,7 +1323,7 @@ var RESTStreamV2 = class extends StreamV2 {
    * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
    * @returns 
    */
-  getUrl(parent) {
+  getNextUrl(previousToken, parent) {
     const urlPattern = [this.baseUrl, this.path].join("");
     return urlPattern;
   }
@@ -2002,20 +1447,20 @@ var StreamWarehouseSyncService = class {
     const columnNamesFromWarehouse = columnsFromWarehouse.map((col) => col.name);
     const columnsMappingStore = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
     const columnsToAdd = Object.keys(columnsMappingStore).filter((unsafeColumnKey) => {
-      const safeColumnName = columnsMappingStore[unsafeColumnKey];
-      if (!safeColumnName) {
+      const safeColumnName2 = columnsMappingStore[unsafeColumnKey];
+      if (!safeColumnName2) {
         return false;
       }
-      return !columnNamesFromWarehouse.includes(safeColumnName);
+      return !columnNamesFromWarehouse.includes(safeColumnName2);
     }).map((unsafeColumnKey) => {
-      const safeColumnName = columnsMappingStore[unsafeColumnKey];
+      const safeColumnName2 = columnsMappingStore[unsafeColumnKey];
       const definition = this.flattenedSchema[unsafeColumnKey];
-      if (!safeColumnName || !definition) {
+      if (!safeColumnName2 || !definition) {
         return void 0;
       }
       return {
         unsafeName: unsafeColumnKey,
-        name: safeColumnName,
+        name: safeColumnName2,
         definition
       };
     }).filter((item) => !!item);
@@ -2230,18 +1675,18 @@ var flattenSchema = (streamId, schema) => {
 
 // src/sdk/models/target/target.ts
 import { Validator } from "jsonschema";
-import moment4 from "moment";
+import dayjs4 from "dayjs";
 
 // src/sdk/models/target/streamDbState.ts
-import moment3 from "moment";
+import dayjs3 from "dayjs";
 
 // src/sdk/models/target/temporaryFile.ts
-import fs3 from "fs-extra";
-import { uniqueId } from "lodash";
+import fs2 from "fs-extra";
+import uniqueId from "lodash/uniqueId.js";
 var createTemporaryFileStream = (streamId) => {
-  fs3.ensureDirSync("./tmp");
+  fs2.ensureDirSync("./tmp");
   const path = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
-  const writeStream = fs3.createWriteStream(path, { flags: "w" });
+  const writeStream = fs2.createWriteStream(path, { flags: "w" });
   return {
     stream: writeStream,
     path
@@ -2289,7 +1734,7 @@ function replaceSymbols(str) {
 }
 
 // src/sdk/models/target/streamDbState.ts
-import * as fs4 from "fs";
+import * as fs3 from "fs";
 var StreamState = class {
   streamId;
   schema;
@@ -2314,7 +1759,7 @@ var StreamState = class {
     this.fileToLoad = createTemporaryFileStream(streamId);
     this.syncedRowCount = 0;
     this.keyProperties = [];
-    this.batchDate = moment3();
+    this.batchDate = dayjs3();
     this.replicationMethod = replicationMethod;
     this._hasBeenLoadedYetDuringThisSync = false;
   }
@@ -2339,7 +1784,7 @@ var StreamState = class {
   resetFileToLoad() {
     this.fileToLoad.stream.close();
     const oldPath = this.fileToLoad.path;
-    fs4.unlinkSync(oldPath);
+    fs3.unlinkSync(oldPath);
     this.fileToLoad = createTemporaryFileStream(this.streamId);
     this.batchedRowCount = 0;
   }
@@ -2365,13 +1810,13 @@ var StreamState = class {
 
 // src/sdk/models/target/target.ts
 import { Semaphore } from "async-mutex";
-import { Promise as BPromise3 } from "bluebird";
+import Bluebird3 from "bluebird";
 var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
   schemaHooks;
-  resolver;
   syncTime;
+  stateProvider;
   // Latest state message received from the Tap
   // Not yet flushed as we didn't upload the stream data since receiving it 
   batchedState;
@@ -2382,15 +1827,15 @@ var ITarget = class _ITarget {
   streams;
   // JSON Schema validator
   validator;
-  constructor(config, resolver) {
+  constructor(config, stateProvider) {
     this.config = config;
-    this.resolver = resolver;
-    this.schemaHooks = resolver.getSchemaHooks();
+    this.stateProvider = stateProvider;
+    this.schemaHooks = [];
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
     this.validator = new Validator();
-    this.syncTime = moment4();
+    this.syncTime = dayjs4();
     this.renameColumnStore = new RenameColumnStore();
   }
   ///////////////////////////////////////////
@@ -2499,7 +1944,7 @@ var ITarget = class _ITarget {
     this.streams[streamId] = new StreamState(
       streamId,
       dbSyncInstance,
-      replicationMethod || "FULL_TABLE"
+      replicationMethod || "FULL_TABLE" /* FULL_TABLE */
     );
   }
   replicationMethod = (message) => {
@@ -2576,41 +2021,9 @@ var ITarget = class _ITarget {
             return translation;
           }
           return k;
-        }),
-        relationships: {
-          left: message.relationships.left.flatMap((l) => {
-            if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
-              const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
-              const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
-              if (from && to) {
-                return [{
-                  type: l.type,
-                  streamId: l.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          }),
-          right: message.relationships.right.flatMap((r) => {
-            if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
-              const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
-              const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
-              if (from && to) {
-                return [{
-                  type: r.type,
-                  streamId: r.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          })
-        }
+        })
       };
-      await BPromise3.map(this.schemaHooks, async (hook) => {
+      await Bluebird3.map(this.schemaHooks, async (hook) => {
         const input = {
           databaseName,
           schemaName,
@@ -2641,9 +2054,6 @@ var ITarget = class _ITarget {
     try {
       logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
       await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
-      const configResolver = loadResolver();
-      await configResolver.markSyncComplete();
-      await configResolver.flushMetrics();
       return Promise.resolve();
     } catch (err) {
       logger.error(`Error while handling complete event.`);
@@ -2686,12 +2096,11 @@ var ITarget = class _ITarget {
     }
   };
   emitState = (state) => {
-    const configResolver = loadResolver();
-    return writeStateFile(configResolver, JSON.stringify(state));
+    return this.stateProvider.writeState(JSON.stringify(state));
   };
   uploadStreamsToWarehouse = async (streamsToUpload) => {
     try {
-      await BPromise3.map(streamsToUpload, async (streamId) => {
+      await Bluebird3.map(streamsToUpload, async (streamId) => {
         logger.info(`\u{1F4E4} Upload stream: ${streamId} to Warehouse`);
         const streamState = this.streams[streamId];
         if (streamState) {
@@ -2725,39 +2134,731 @@ var ITarget = class _ITarget {
     }
   };
 };
+
+// src/targets/bigquery/helpers.ts
+var safeColumnName = (key) => {
+  let returnString = key;
+  const shouldNotStartWith = ["_TABLE_", "_FILE_", "_PARTITION"];
+  shouldNotStartWith.forEach((snm) => {
+    if (returnString.startsWith(snm)) {
+      returnString = returnString.replace(snm, "");
+    }
+  });
+  returnString = returnString.replace(/^[-\d\s]*/g, "");
+  returnString = returnString.toLowerCase();
+  returnString = returnString.replace("`", "");
+  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+  if (returnString.length > 300) {
+    returnString = returnString.substr(0, 300);
+  }
+  const pattern = /[^a-zA-Z0-9_]/gm;
+  return returnString.replace(pattern, "_");
+};
+var safeTableName = (key) => {
+  let returnString = key;
+  returnString = returnString.toLowerCase();
+  returnString = returnString.replace("`", "");
+  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+  if (returnString.length > 1024) {
+    returnString = returnString.substr(0, 1024);
+  }
+  const pattern = /[^a-zA-Z0-9_]/gm;
+  return returnString.replace(pattern, "_");
+};
+
+// src/targets/bigquery/service/record.ts
+import Decimal from "decimal.js";
+import dayjs5 from "dayjs";
+var validateDateRange = (record, schema) => {
+  Object.keys(schema).forEach((key) => {
+    const { format: format6 } = schema[key];
+    if (format6 === "date-time") {
+      if (record[key]) {
+        const formattedDate = dayjs5(record[key]).format(defaultDateTimeFormat);
+        const isNotTooMuchInTheFuture = dayjs5(record[key]).isBefore(dayjs5("9999-12-31 23:59:59.999999"));
+        const isNotTooMuchInThePast = dayjs5(record[key]).isAfter(dayjs5("0001-01-01 00:00:00"));
+        if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
+          record[key] = formattedDate;
+        } else {
+          record[key] = void 0;
+          logger.info("Skipping date for key %s for being invalid", key);
+        }
+      }
+    }
+  });
+  return record;
+};
+var maxBQNumericValue = new Decimal("9.9999999999999999999999999999999999999E+28");
+var convertNumberIntoDecimal = (record, schema) => {
+  Object.keys(record).forEach((key) => {
+    if (schema[key]) {
+      const { type, items } = schema[key];
+      const types = type instanceof Array ? type : [type];
+      if (types.includes("number")) {
+        if (typeof record[key] === "number" || record[key] instanceof Decimal) {
+          const decimal = new Decimal(record[key]).toDecimalPlaces(9);
+          if (decimal.greaterThanOrEqualTo(maxBQNumericValue)) {
+            record[key] = null;
+          } else {
+            record[key] = decimal;
+          }
+        } else {
+          record[key] = null;
+        }
+      } else if (types.includes("array") && items?.type.includes("number")) {
+        if (Array.isArray(record[key])) {
+          const rawArr = record[key];
+          const newArr = rawArr.map((item) => {
+            if (typeof item === "number" || item instanceof Decimal) {
+              const parsed = new Decimal(item).toDecimalPlaces(9);
+              if (parsed.greaterThanOrEqualTo(maxBQNumericValue)) {
+                return void 0;
+              } else {
+                return parsed;
+              }
+            } else {
+              return void 0;
+            }
+          }).filter((item) => !!item);
+          record[key] = newArr;
+        } else {
+          record[key] = null;
+        }
+      }
+    }
+  });
+  return record;
+};
+
+// src/targets/bigquery/service/dbSync.ts
+import dayjs6 from "dayjs";
+
+// src/targets/bigquery/service/bigquery.ts
+import { BigQuery } from "@google-cloud/bigquery";
+var BqClientHolder = class {
+  static instance;
+  client;
+  constructor(config) {
+    const retryOptions = {
+      autoRetry: true,
+      maxRetries: 3
+    };
+    this.client = new BigQuery({
+      projectId: config.database.trim(),
+      ...retryOptions
+    });
+  }
+  static client(config) {
+    if (!this.instance) {
+      this.instance = new this(config);
+    }
+    return this.instance.client;
+  }
+};
+
+// src/targets/bigquery/service/cloudStorage.ts
+import { Storage } from "@google-cloud/storage";
+
+// node_modules/uuid/dist/esm-node/rng.js
+import crypto from "crypto";
+var rnds8Pool = new Uint8Array(256);
+var poolPtr = rnds8Pool.length;
+function rng() {
+  if (poolPtr > rnds8Pool.length - 16) {
+    crypto.randomFillSync(rnds8Pool);
+    poolPtr = 0;
+  }
+  return rnds8Pool.slice(poolPtr, poolPtr += 16);
+}
+
+// node_modules/uuid/dist/esm-node/regex.js
+var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
+
+// node_modules/uuid/dist/esm-node/validate.js
+function validate(uuid) {
+  return typeof uuid === "string" && regex_default.test(uuid);
+}
+var validate_default = validate;
+
+// node_modules/uuid/dist/esm-node/stringify.js
+var byteToHex = [];
+for (let i = 0; i < 256; ++i) {
+  byteToHex.push((i + 256).toString(16).substr(1));
+}
+function stringify3(arr, offset = 0) {
+  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
+  if (!validate_default(uuid)) {
+    throw TypeError("Stringified UUID is invalid");
+  }
+  return uuid;
+}
+var stringify_default = stringify3;
+
+// node_modules/uuid/dist/esm-node/v4.js
+function v4(options, buf, offset) {
+  options = options || {};
+  const rnds = options.random || (options.rng || rng)();
+  rnds[6] = rnds[6] & 15 | 64;
+  rnds[8] = rnds[8] & 63 | 128;
+  if (buf) {
+    offset = offset || 0;
+    for (let i = 0; i < 16; ++i) {
+      buf[offset + i] = rnds[i];
+    }
+    return buf;
+  }
+  return stringify_default(rnds);
+}
+var v4_default = v4;
+
+// src/targets/bigquery/service/cloudStorage.ts
+var uploadFileInBucket = async (streamId, gcsBuckerName, connectorId, filePath) => {
+  try {
+    const retryOptions = { autoRetry: true, maxRetries: 20 };
+    let storage = new Storage({ retryOptions });
+    const bucketRef = storage.bucket(gcsBuckerName);
+    await bucketRef.get({ autoCreate: false });
+    const destinationFileName = `${connectorId}/${streamId}-${v4_default()}.jsonnl`;
+    await bucketRef.upload(filePath, {
+      destination: destinationFileName
+    });
+    logger.info(`\u{1F5F3}  Uploaded ${filePath} into ${gcsBuckerName}/${destinationFileName} GCS File`);
+    return bucketRef.file(destinationFileName);
+  } catch (err) {
+    logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
+        
+        Error: ${err.message}
+        Stack: ${err.stack}
+        Code: ${err.code}
+        `);
+    if (err.code < 500) {
+      await haltAndCatchFire(
+        `unauthorized`,
+        `We couldn't connect to your Cloud Storage bucket \u{1F614}
+                    
+                    The error from Google is: '${err.message}' \u{1F440}
+                    
+                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
+        `Got error from cloud storage lib`
+      );
+    }
+    throw err;
+  }
+};
+
+// src/targets/bigquery/service/dbSync.ts
+import chunk from "lodash/chunk.js";
+import { Semaphore as Semaphore2 } from "async-mutex";
+var semaphore2 = new Semaphore2(1);
+var BigQueryDBSync = class extends StreamWarehouseSyncService {
+  config;
+  bqClient;
+  bqDataset;
+  bqTable;
+  bqStagingTable;
+  tableName;
+  stagingTableName;
+  sqlTableId;
+  sqlStagingTableId;
+  constructor(config, streamId, database, schema, table, renameColumnStore) {
+    super(
+      streamId,
+      database,
+      schema,
+      table,
+      renameColumnStore
+    );
+    this.config = config;
+    this.bqClient = BqClientHolder.client(config);
+    this.bqDataset = this.bqClient.dataset(schema, { location: config.location || "EU" });
+    this.bqTable = this.bqDataset.table(table, { location: config.location || "EU" });
+    this.tableName = table;
+    this.stagingTableName = `${table}_temp`;
+    this.bqStagingTable = this.bqDataset.table(this.stagingTableName, { location: config.location || "EU" });
+    this.sqlTableId = `${schema}.${table}`;
+    this.sqlStagingTableId = `${this.sqlTableId}_temp`;
+  }
+  // Implementation of abstract classes
+  getSerializedRecord = (record) => {
+    return JSON.stringify(record) + `
+`;
+  };
+  safeColumnName = safeColumnName;
+  getWarehouseTypeFromJSONSchema = (definition) => {
+    const type = definition.type;
+    const format6 = definition.format;
+    let typeArr;
+    if (type instanceof Array) {
+      typeArr = type;
+    } else {
+      typeArr = [type];
+    }
+    if (format6) {
+      switch (format6) {
+        case "date-time":
+          return "TIMESTAMP";
+        case "json":
+          return "STRING";
+        default:
+          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format6}`);
+      }
+    } else if (typeArr.includes("number")) {
+      return "NUMERIC";
+    } else if (typeArr.includes("integer") && type.includes("string")) {
+      return "NUMERIC";
+    } else if (typeArr.includes("integer")) {
+      return "INTEGER";
+    } else if (typeArr.includes("boolean")) {
+      return "BOOLEAN";
+    } else if (typeArr.includes("string")) {
+      return "STRING";
+    } else if (typeArr.includes("array")) {
+      const items = definition.items;
+      if (items) {
+        return this.getWarehouseTypeFromJSONSchema(items);
+      }
+      throw new Error(`StreamId: ${this.streamId} -  Unsupported array schema field definition: ${JSON.stringify(definition)}`);
+    } else {
+      throw new Error(`StreamId: ${this.streamId} -  Unsupported schema field definition: ${JSON.stringify(definition)}`);
+    }
+  };
+  getMergeQueries = () => {
+    const primaryKeyCondition = this.primaryKeys.map((pk) => {
+      return `\`sourced\`.\`${pk}\` = \`destination\`.\`${pk}\``;
+    }).join(` AND `);
+    const primaryKeys = this.primaryKeys.map((pk) => `\`${pk}\``).join(` ,`);
+    const columnUnsafeToSafeMapping = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    const columns = Object.keys(columnUnsafeToSafeMapping).map((column) => {
+      return {
+        isUnsafe: true,
+        colName: column
+      };
+    });
+    const columnsChunks = chunk(columns, 300).map((chunk2) => {
+      this.primaryKeys.forEach((pk) => {
+        if (!chunk2.map((c) => columnUnsafeToSafeMapping[c.colName]).includes(pk)) {
+          chunk2.push({
+            isUnsafe: false,
+            colName: pk
+          });
+        }
+      });
+      return chunk2;
+    });
+    return columnsChunks.map((chunk2) => {
+      const setValues = chunk2.map((column) => {
+        if (column.isUnsafe) {
+          return `\`destination\`.\`${columnUnsafeToSafeMapping[column.colName]}\` = \`sourced\`.\`${columnUnsafeToSafeMapping[column.colName]}\``;
+        } else {
+          return `\`destination\`.\`${column.colName}\` = \`sourced\`.\`${column.colName}\``;
+        }
+      }).join(`, `);
+      const renamedCols = chunk2.map((column) => {
+        if (column.isUnsafe) {
+          return `\`${columnUnsafeToSafeMapping[column.colName]}\``;
+        } else {
+          return `\`${column.colName}\``;
+        }
+      }).join(`, `);
+      const query = `
+        MERGE ${this.sqlTableId} destination
+        USING (
+            WITH all_staging_rows AS (
+                SELECT *
+                FROM ${this.sqlStagingTableId}
+            ),
+            numbered_rows AS (
+                SELECT *, 
+                    ROW_NUMBER() OVER (PARTITION BY ${primaryKeys}) as _wly_row_nb,
+                    COUNT(1) OVER (PARTITION BY ${primaryKeys}) AS _wly_partition_size,
+                FROM all_staging_rows
+            ),
+            staging_deduped AS (
+                SELECT * EXCEPT(_wly_partition_size, _wly_row_nb)
+                FROM numbered_rows 
+                WHERE _wly_partition_size = _wly_row_nb
+            )
+            SELECT *
+            FROM staging_deduped
+        ) AS sourced
+        ON ${primaryKeyCondition}
+        WHEN MATCHED THEN
+            UPDATE SET ${setValues}
+        WHEN NOT MATCHED THEN
+            INSERT (${renamedCols}) VALUES (${renamedCols});
+    `;
+      return query;
+    });
+  };
+  getReplaceQueries = () => {
+    return [
+      `TRUNCATE TABLE ${this.sqlTableId};`,
+      ...this.getMergeQueries()
+    ];
+  };
+  createDatabaseAndSchemaIfNotExists = async (retryCount) => {
+    try {
+      await semaphore2.runExclusive(async () => {
+        const [exists] = await this.bqDataset.exists();
+        if (!exists) {
+          logger.info(`\u{1F423} Creating dataset: ${this.bqDataset.id} on BigQuery`);
+          await this.bqDataset.create();
+        } else {
+          const [metadata] = await this.bqDataset.getMetadata();
+          const alreadyExistinglocation = metadata.location;
+          if (this.config.location && this.config.location !== alreadyExistinglocation) {
+            await haltAndCatchFire(
+              `unknown`,
+              `A BigQuery dataset called '${this.bqDataset.id}' already exists in location: '${alreadyExistinglocation}' \u{1F605}
+                            
+                            This connector is configured to load data into the location '${this.config.location}' which is not possible as 2 datasets sharing the same name can't exist in different location on Google Cloud \u{1F63F}
+
+                            To fix this issue, could you either: a. Delete the already existing dataset? b. Change the configured destination schema name to avoid the name conflict? \u{1F64F}
+                            
+                            Please relaunch the connector once it's done to fix the issue \u{1F917}
+                            `,
+              `
+                            We have a conflict between an already existing dataset in location ${alreadyExistinglocation} and the configured location: ${this.config.location}
+                            `
+            );
+            throw Error();
+          }
+          logger.info(`\u{1F44B} Dataset: ${this.bqDataset.id} already exists on BigQuery, reusing it.`);
+        }
+      });
+    } catch (err) {
+      if (err.code === 400) {
+        await haltAndCatchFire(
+          `unauthorized`,
+          `We can't connect to your BigQuery account \u{1F614}
+
+                    Here is the message from BigQuery: ${err.message}
+                    
+                    Can you check that your Warehouse credentials and Source target schema are properly configured and try to sync again the source? \u{1F64F}
+                    `,
+          `Got error when trying to get Dataset from BigQuery: ${err.message} - ${err.code}`
+        );
+        return;
+      }
+      if (retryCount && retryCount > this.maxRetryCount) {
+        logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. ${err.message} - ${err.code}`);
+        gracefulExit(1);
+      } else {
+        logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. Retrying... ${err.message} - ${err.code}`);
+        await new Promise((resolve, reject) => setTimeout(resolve, 1e3 * retryCount));
+        await this.createDatabaseAndSchemaIfNotExists(retryCount + 1);
+      }
+    }
+    return Promise.resolve();
+  };
+  createTable = async () => {
+    try {
+      const tableName = this.tableName;
+      const schema = this.generateBigquerySchema();
+      logger.info(`\u{1F423} Creating table: ${tableName} on BigQuery`);
+      const createTableOptions = {
+        schema
+      };
+      await this.bqDataset.createTable(
+        tableName,
+        createTableOptions
+      );
+      return;
+    } catch (err) {
+      throw new Error(`StreamId: ${this.streamId} - Issue when creating table in BigQuery
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+  addColumns = async (tableFields) => {
+    if (tableFields.length > 0) {
+      logger.info(`\u{1F195} Stream: ${this.streamId} - Adding following columns in BigQuery schema: ${JSON.stringify(tableFields)}`);
+      const [metadata] = await this.bqTable.getMetadata();
+      const bgTableFields = tableFields.map((tblField) => {
+        return this.getBigqueryTableFieldFromInternalSchemaFieldDefinition(tblField.name, tblField.definition);
+      }).filter((fld) => {
+        return !!fld;
+      });
+      const schema = metadata.schema;
+      const new_schema = {
+        ...schema,
+        fields: schema.fields.concat(bgTableFields)
+      };
+      metadata.schema = new_schema;
+      await this.bqTable.setMetadata(metadata);
+      return;
+    }
+  };
+  createStagingArea = async () => {
+    try {
+      const tableName = this.stagingTableName;
+      const [exists] = await this.bqDataset.table(tableName).exists();
+      if (exists) {
+        logger.info(`\u{1F474} Temporary table ${tableName} already exists, it should be a relic form the past.
+                Dropping it.`);
+        await this.deleteStagingArea();
+      }
+      const schema = this.generateBigquerySchema();
+      const expirationTime = dayjs6().add(1, "day").valueOf().toString();
+      const createTableOptions = {
+        schema,
+        expirationTime
+      };
+      logger.info(`\u{1F423} Creating staging table: ${tableName} on BigQuery`);
+      await this.bqDataset.createTable(
+        tableName,
+        createTableOptions
+      );
+      return;
+    } catch (err) {
+      throw new Error(`StreamId: ${this.streamId} - Issue when creating table in BigQuery
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+  loadStreamInStagingArea = async (localFilePath) => {
+    try {
+      const file = await uploadFileInBucket(
+        this.streamId,
+        this.config.loading_deck_gcs_bucket_name,
+        this.config.connector_id,
+        localFilePath
+      );
+      const metadata = {
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        writeDisposition: "WRITE_TRUNCATE"
+      };
+      await this.loadGCSFileInTable(this.sqlStagingTableId, file, metadata);
+    } catch (err) {
+      logger.error(`StreamId: ${this.streamId} - Error while uploading stream.`);
+      throw err;
+    }
+  };
+  deleteStagingArea = () => {
+    const query = `DROP TABLE IF EXISTS ${this.sqlStagingTableId};`;
+    return this.runQueriesWithRetry([query]);
+  };
+  getTablesInSchema = async () => {
+    const [bqTables] = await this.bqDataset.getTables();
+    const foundTables = bqTables.filter((tbl) => {
+      return !!tbl.id;
+    }).map((bqTable) => {
+      return {
+        tableName: bqTable.id
+      };
+    });
+    return foundTables;
+  };
+  getTableColumnsFromWarehouse = async () => {
+    const [table] = await this.bqTable.getMetadata();
+    return table.schema.fields;
+  };
+  runQueries = async (queries) => {
+    const sqlQuery = `
+        BEGIN
+            ${queries.join(`
+            `)}
+        END;
+    `;
+    await this.bqClient.query(sqlQuery);
+  };
+  // Specific methods
+  /**
+   * 
+   * Generate a TableField definition object that can be passed to BigQuery APIs to define a column on a table
+   * 
+   * @param colName 
+   * @param schemaFieldDefinition 
+   * @returns 
+   */
+  getBigqueryTableFieldFromInternalSchemaFieldDefinition = (colName, schemaFieldDefinition) => {
+    const name = colName;
+    if (!schemaFieldDefinition) {
+      throw new Error(`StreamId: ${this.streamId} - Schema is unknown for colName=${colName}, so we can't infer the proper BigQuery definition for this column.`);
+    }
+    const type = schemaFieldDefinition.type;
+    let typeArr;
+    if (type instanceof Array) {
+      typeArr = type;
+    } else {
+      typeArr = [type];
+    }
+    if (typeArr.includes("array")) {
+      return {
+        name,
+        mode: "REPEATED",
+        type: this.getWarehouseTypeFromJSONSchema(schemaFieldDefinition)
+      };
+    } else {
+      let nullableMode = "NULLABLE";
+      return {
+        name,
+        mode: nullableMode,
+        type: this.getWarehouseTypeFromJSONSchema(schemaFieldDefinition)
+      };
+    }
+  };
+  generateBigquerySchema = () => {
+    const unsafeToSafeMappingColumnsFromColumnStore = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    if (!unsafeToSafeMappingColumnsFromColumnStore) {
+      throw new Error(`StreamId: ${this.streamId} - There is no unsafe->safe column mapping`);
+    }
+    return Object.keys(unsafeToSafeMappingColumnsFromColumnStore).reduce((acc, unsafeKey) => {
+      const renamedColName = this.renamedColumnStore.getColumnTranslation(this.streamId, unsafeKey);
+      const bigqueryTableField = this.getBigqueryTableFieldFromInternalSchemaFieldDefinition(renamedColName, this.flattenedSchema[unsafeKey]);
+      if (bigqueryTableField) {
+        acc.push(bigqueryTableField);
+      }
+      return acc;
+    }, []);
+  };
+  loadGCSFileInTable = async (tableName, file, metadata) => {
+    try {
+      const [job] = await this.bqStagingTable.load(file, metadata);
+      logger.info(`\u{1F69A} Loaded GCS file \`${file.id}\` in BigQuery table: ${tableName} with Job: \`${job.id}\``);
+      logger.debug(`Upload Stats: ${JSON.stringify(job.statistics)}`);
+      const errors2 = job.status?.errors;
+      if (errors2 && errors2.length > 0) {
+        throw errors2;
+      }
+    } catch (err) {
+      logger.error(err);
+      throw new Error(`StreamId: ${this.streamId} - Issue when loading file in BigQuery table | fileName: ${file.name}, tableName: ${tableName}
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+};
+
+// src/targets/bigquery/main.ts
+var BigQueryTarget = class extends ITarget {
+  constructor(config, stateProvider) {
+    super(config, stateProvider);
+    this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
+  }
+  static requiredConfigKeys = [
+    "schema",
+    "project_id",
+    "loading_deck_gcs_bucket_name"
+  ];
+  newDBSyncInstance = (streamId, database, schema, table) => {
+    return new BigQueryDBSync(
+      this.config,
+      streamId,
+      database,
+      schema,
+      table,
+      this.renameColumnStore
+    );
+  };
+  genTableName = safeTableName;
+  safeColumnNameConverter = safeColumnName;
+  validateDateRange = validateDateRange;
+  convertNumberIntoDecimal = convertNumberIntoDecimal;
+};
+
+// src/sdk/service/gcs.ts
+import { Storage as Storage2 } from "@google-cloud/storage";
+import { format as format4 } from "util";
+var getStorage = () => new Storage2({ retryOptions: { autoRetry: true, maxRetries: 20 } });
+var readObjectAsString = async (bucketName, objectPath) => {
+  try {
+    const storage = getStorage();
+    const bucketRef = storage.bucket(bucketName);
+    await bucketRef.get({ autoCreate: false });
+    const fileRef = bucketRef.file(objectPath);
+    const [exists] = await fileRef.exists();
+    if (!exists) {
+      throw new Error(`GCS object not found: gs://${bucketName}/${objectPath}`);
+    }
+    const [contents] = await fileRef.download();
+    return contents.toString("utf8");
+  } catch (err) {
+    throw new Error(format4(`error reading GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
+  }
+};
+var writeStringObject = async (bucketName, objectPath, contents) => {
+  try {
+    const storage = getStorage();
+    const bucketRef = storage.bucket(bucketName);
+    await bucketRef.get({ autoCreate: false });
+    const fileRef = bucketRef.file(objectPath);
+    await fileRef.save(contents, { contentType: "application/json" });
+    logger.info(`\u{1F5F3}  Uploaded object to gs://${bucketName}/${objectPath}`);
+  } catch (err) {
+    throw new Error(format4(`error writing GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
+  }
+};
+
+// src/state-providers/gcs/main.ts
+import { format as format5 } from "util";
+var GCSStateProvider = class {
+  bucketName;
+  connectorId;
+  constructor(connectorId, bucketName) {
+    this.connectorId = connectorId;
+    this.bucketName = bucketName;
+  }
+  getObjectPath() {
+    return `${this.connectorId}/state.json`;
+  }
+  async getState() {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
+      const raw = await readObjectAsString(this.bucketName, objectPath);
+      const parsed = JSON.parse(raw);
+      return { state: parsed };
+    } catch (err) {
+      logger.warn(format5(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
+      return { state: void 0 };
+    }
+  }
+  async writeState(state) {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
+      await writeStringObject(this.bucketName, objectPath, state);
+    } catch (err) {
+      logger.error(format5(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+      throw new Error(format5(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+};
 export {
   ALL_STREAM_ID_KEYWORD,
   API_CALLS_METRIC_NAME,
   Authenticator,
   BATCH_INTERVAL_MS,
-  Catalog,
+  BigQueryTarget,
   CounterMetric,
   DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME,
+  GCSStateProvider,
   ITarget,
   MissingFieldInSchemaError,
   MissingSchemaError,
-  RESTStreamV2,
+  RESTStream,
   ROWS_SYNCED_METRIC_NAME,
   RecordMessage,
   RenameColumnStore,
+  ReplicationMethod,
   ReplicationMethodMessage,
-  Resolver,
   SchemaMessage,
   SchemaValidationError,
   StateMessage,
   StateService,
-  StreamMetadata,
-  StreamV2,
+  Stream,
   StreamWarehouseSyncService,
   Tap,
   _greaterThan,
   addWhalyFields,
-  checkRequiredConfigKeys,
   createTemporaryFileStream,
   extractStateForStream,
   finalizeStateProgressMarkers,
-  findRelatedRelationships,
   flattenSchema,
   getAllMetrics,
   getAxiosInstance,
@@ -2772,15 +2873,12 @@ export {
   incrementStreamState,
   loadJson,
   loadSchema,
-  loadShoreConfig,
   logger,
   postFormDataApiCall,
   postJSONApiCall,
   postUrlEncodedApiCall,
   printMemoryFootprint,
   removeParasiteProperties,
-  safePath,
-  writeCatalogFile,
-  writeStateFile
+  safePath
 };
 //# sourceMappingURL=index.mjs.map