@whaly/connector-sdk 0.1.0

package/dist/index.js

@@ -0,0 +1,2878 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ALL_STREAM_ID_KEYWORD: () => ALL_STREAM_ID_KEYWORD,
+  API_CALLS_METRIC_NAME: () => API_CALLS_METRIC_NAME,
+  Authenticator: () => Authenticator,
+  BATCH_INTERVAL_MS: () => BATCH_INTERVAL_MS,
+  Catalog: () => Catalog,
+  CounterMetric: () => CounterMetric,
+  DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
+  EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
+  ITarget: () => ITarget,
+  MissingFieldInSchemaError: () => MissingFieldInSchemaError,
+  MissingSchemaError: () => MissingSchemaError,
+  RESTStreamV2: () => RESTStreamV2,
+  ROWS_SYNCED_METRIC_NAME: () => ROWS_SYNCED_METRIC_NAME,
+  RecordMessage: () => RecordMessage,
+  RenameColumnStore: () => RenameColumnStore,
+  ReplicationMethodMessage: () => ReplicationMethodMessage,
+  Resolver: () => Resolver,
+  SchemaMessage: () => SchemaMessage,
+  SchemaValidationError: () => SchemaValidationError,
+  StateMessage: () => StateMessage,
+  StateService: () => StateService,
+  StreamMetadata: () => StreamMetadata,
+  StreamV2: () => StreamV2,
+  StreamWarehouseSyncService: () => StreamWarehouseSyncService,
+  Tap: () => Tap,
+  _greaterThan: () => _greaterThan,
+  addWhalyFields: () => addWhalyFields,
+  checkRequiredConfigKeys: () => checkRequiredConfigKeys,
+  createTemporaryFileStream: () => createTemporaryFileStream,
+  extractStateForStream: () => extractStateForStream,
+  finalizeStateProgressMarkers: () => finalizeStateProgressMarkers,
+  findRelatedRelationships: () => findRelatedRelationships,
+  flattenSchema: () => flattenSchema,
+  getAllMetrics: () => getAllMetrics,
+  getAxiosInstance: () => getAxiosInstance,
+  getCounterMetric: () => getCounterMetric,
+  getCounterMetrics: () => getCounterMetrics,
+  getDownloadFileApiCall: () => getDownloadFileApiCall,
+  getJSONApiCall: () => getJSONApiCall,
+  getJSONApiCallWithFullResponse: () => getJSONApiCallWithFullResponse,
+  getRawJSONApiCall: () => getRawJSONApiCall,
+  gracefulExit: () => gracefulExit,
+  haltAndCatchFire: () => haltAndCatchFire,
+  incrementStreamState: () => incrementStreamState,
+  loadJson: () => loadJson,
+  loadSchema: () => loadSchema,
+  loadShoreConfig: () => loadShoreConfig,
+  logger: () => logger,
+  postFormDataApiCall: () => postFormDataApiCall,
+  postJSONApiCall: () => postJSONApiCall,
+  postUrlEncodedApiCall: () => postUrlEncodedApiCall,
+  printMemoryFootprint: () => printMemoryFootprint,
+  removeParasiteProperties: () => removeParasiteProperties,
+  safePath: () => safePath,
+  writeCatalogFile: () => writeCatalogFile,
+  writeStateFile: () => writeStateFile
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/sdk/utils.ts
+var import_fs2 = require("fs");
+
+// src/sdk/service/logger.ts
+var winston2 = __toESM(require("winston"));
+
+// src/sdk/models/resolver.ts
+var Resolver = class {
+  constructor() {
+  }
+};
+
+// src/resolvers/local-file/services/files.ts
+var import_fs = __toESM(require("fs"));
+var writeFile = (path, content) => {
+  return Promise.resolve(import_fs.default.writeFileSync(path, content));
+};
+var readAndParseJSONFile = (path) => {
+  if (!import_fs.default.existsSync(path)) {
+    throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
+  }
+  const raw = import_fs.default.readFileSync(path);
+  const parsed = JSON.parse(raw.toString());
+  return parsed;
+};
+
+// src/resolvers/local-file/hook/schemaHook.ts
+var import_fs_extra = require("fs-extra");
+var LocalSchemaHook = class {
+  async writeSchema(input) {
+    await (0, import_fs_extra.appendFile)("./schema.tmp", JSON.stringify(input) + "\n");
+  }
+};
+
+// src/resolvers/local-file/resolver.ts
+var import_winston = __toESM(require("winston"));
+
+// src/sdk/service/metric.ts
+var metricStore = {};
+var ALL_STREAM_ID_KEYWORD = "all";
+var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+var API_CALLS_METRIC_NAME = "api_calls_count";
+var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+var getCounterMetrics = (name, streamIds) => {
+  return streamIds.map((streamId) => {
+    return getCounterMetric(name, streamId);
+  });
+};
+var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
+  if (!metricStore[streamId]) {
+    metricStore[streamId] = {};
+  }
+  const streamMetricStore = metricStore[streamId];
+  if (!streamMetricStore[name]) {
+    streamMetricStore[name] = new CounterMetric(name, streamId);
+  }
+  return streamMetricStore[name];
+};
+var getAllMetrics = () => {
+  return Object.keys(metricStore).flatMap((streamId) => {
+    const streamMetricStore = metricStore[streamId];
+    if (!streamMetricStore) {
+      return [];
+    }
+    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
+  });
+};
+var CounterMetric = class {
+  value = 0;
+  name;
+  streamId;
+  constructor(name, streamId) {
+    this.name = name;
+    this.streamId = streamId;
+  }
+  increment(inc = 1) {
+    this.value = this.value + inc;
+  }
+  getStreamId() {
+    return this.streamId;
+  }
+  getName() {
+    return this.name;
+  }
+  getValue() {
+    return this.value;
+  }
+};
+
+// src/resolvers/local-file/resolver.ts
+var { combine, prettyPrint, timestamp, errors, splat } = import_winston.default.format;
+var configPath = "./config.json";
+var targetConfigPath = "./target-config.json";
+var catalogPath = "./catalog.json";
+var statePath = "./state.json";
+var metricsPath = "./metrics.json";
+var LocalFilesResolver = class extends Resolver {
+  startVPNIfNeeded() {
+    return Promise.resolve();
+  }
+  checkIfCanSync() {
+    return Promise.resolve(true);
+  }
+  markSyncStarted() {
+    return Promise.resolve();
+  }
+  onError(errorType, errorText, errorDebugText) {
+    const path = `./error.json`;
+    return writeFile(path, JSON.stringify({
+      errorType,
+      errorText,
+      errorDebugText
+    }));
+  }
+  getSchemaHooks() {
+    const wlyHook = new LocalSchemaHook();
+    return [wlyHook];
+  }
+  writeSchema() {
+    return Promise.resolve();
+  }
+  constructor() {
+    super();
+  }
+  getConfig(command) {
+    logger.info(`\u{1F4C1} Using the LOCAL resolver`);
+    const config = readAndParseJSONFile(configPath);
+    const targetConfig = readAndParseJSONFile(targetConfigPath);
+    const destination = process.env.SHORE__DESTINATION;
+    if (!destination) {
+      throw new Error(`\u274C Env variable \`SHORE__DESTINATION\` is not set.
+            Did you forget to configure it?
+            
+            Possible values are: \`GOOGLE_BIGQUERY\`, \`SNOWFLAKE\``);
+    }
+    if (command === `READ`) {
+      if (destination === `GOOGLE_BIGQUERY`) {
+        const googleCredentials = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+        if (!googleCredentials) {
+          throw new Error(`Env variable \`GOOGLE_APPLICATION_CREDENTIALS\` is not set. 
+                Did you forget to configure it?
+
+                This variable should link to a file containing the proper Google Service Account credentials.
+                `);
+        }
+      }
+      const catalog = readAndParseJSONFile(catalogPath);
+      const state = readAndParseJSONFile(statePath);
+      return Promise.resolve({
+        command,
+        destination,
+        config,
+        targetConfig,
+        catalog,
+        state
+      });
+    } else {
+      return Promise.resolve({
+        command,
+        destination,
+        config,
+        targetConfig
+      });
+    }
+  }
+  writeCatalog(catalog) {
+    return writeFile(catalogPath, JSON.stringify(catalog));
+  }
+  writeState(state) {
+    return writeFile(statePath, state);
+  }
+  markSyncFailed() {
+    logger.info(`\u{1F622} Sync has failed.`);
+    return Promise.resolve();
+  }
+  markSyncComplete() {
+    logger.info(`\u{1F389} Sync is complete.`);
+    return Promise.resolve();
+  }
+  markDiscoveryComplete() {
+    logger.info(`\u{1F389} Discovery is complete.`);
+    return Promise.resolve();
+  }
+  async flushMetrics() {
+    const allMetrics = getAllMetrics();
+    const metricsOuput = allMetrics.map((metric) => {
+      return {
+        streamId: metric.getStreamId(),
+        name: metric.getName(),
+        value: metric.getValue()
+      };
+    }).map((obj) => JSON.stringify(obj)).join(`
+`);
+    return writeFile(metricsPath, metricsOuput);
+  }
+  updateSourceValue(optionKey, optionValue) {
+    logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
+    const config = readAndParseJSONFile(configPath);
+    config[optionKey] = optionValue;
+    return writeFile(configPath, JSON.stringify(config));
+  }
+  getLogFormat() {
+    return combine(
+      timestamp(),
+      errors({ stack: true }),
+      splat(),
+      prettyPrint()
+    );
+  }
+};
+
+// src/sdk/service/resolver.ts
+function initResolver(resolverType) {
+  if (resolverType === "LOCAL") {
+    return new LocalFilesResolver();
+  } else {
+    throw new Error(`\u274C Unsupported configResolver: ${resolverType}`);
+  }
+}
+var loadResolver = () => {
+  const configResolver = process.env.SHORE__CONFIG_RESOLVER;
+  if (!configResolver) {
+    throw new Error(`Env variable \`SHORE__CONFIG_RESOLVER\` is not set. 
+        Did you forget to configure it?
+
+        Possible values are \`LOCAL\`, \`WHALY\`.
+        Please read the documentation to get the full description of the behavior of each.
+        `);
+  }
+  if (configResolver === "WHALY" || configResolver === "LOCAL") {
+    return initResolver(configResolver);
+  } else {
+    throw new Error(`Resolver: ${configResolver} is not supported.
+        Did you properly configure \`SHORE__CONFIG_RESOLVER\`?`);
+  }
+};
+
+// src/sdk/service/logger.ts
+var BATCH_INTERVAL_MS = 100;
+var transports2 = {
+  console: new winston2.transports.Console({
+    stderrLevels: [],
+    level: "info"
+  })
+};
+var getTransports = () => {
+  return [
+    transports2.console
+  ];
+};
+var winstonLogger = winston2.createLogger({
+  format: loadResolver().getLogFormat(),
+  transports: getTransports()
+});
+var isWinstonOpen = true;
+var closeWinston = (cb) => {
+  isWinstonOpen = false;
+  winstonLogger.end();
+  winstonLogger.on("finish", () => {
+    setTimeout(() => {
+      return cb();
+    }, BATCH_INTERVAL_MS * 2);
+  });
+};
+var log = (level, message, ...args) => {
+  if (isWinstonOpen) {
+    winstonLogger[level](message, ...args);
+  } else {
+    console[level](message, ...args);
+  }
+};
+var logger = {
+  info: (message, ...args) => log("info", message, ...args),
+  error: (message, ...args) => log("error", message, ...args),
+  debug: (message, ...args) => log("debug", message, ...args),
+  warn: (message, ...args) => log("warn", message, ...args),
+  closeWinston
+};
+
+// src/sdk/utils.ts
+var writeStateFile = (resolver, state) => {
+  logger.info(`\u{1F4DD} Writing the state file.`);
+  return resolver.writeState(state);
+};
+var writeCatalogFile = (resolver, catalog) => {
+  logger.info(`\u{1F4DD} Writing the catalog`);
+  return resolver.writeCatalog(catalog);
+};
+var loadShoreConfig = (resolver) => {
+  const command = process.env.SHORE__COMMAND;
+  if (!command) {
+    throw new Error(`Env variable \`SHORE__COMMAND\` is not set. 
+        Did you forget to configure it?
+
+        Possible values are: \`DISCOVER\`, \`READ\``);
+  }
+  if (command === `DISCOVER`) {
+    logger.info(`\u{1F50E} Mode: discovery
+        Shore will detect the available streams in the source and their configuration.`);
+  } else if (command === `READ`) {
+    logger.info(`\u{1F501} Mode: Synchronizing data
+        Shore will extract and push data from the source to the destination.`);
+  } else {
+    throw new Error(`Command: \`${command}\` is not supported.
+        Did you properly configured env variable \`SHORE__COMMAND\`?`);
+  }
+  return resolver.getConfig(command);
+};
+function readFile(fileName, filePath) {
+  try {
+    return (0, import_fs2.readFileSync)(filePath);
+  } catch (err) {
+    logger.error(`Can't open file: \`${fileName}\` at path: \`${filePath}\`. Is it at the proper place?`);
+    throw err;
+  }
+}
+function parseFileContent(fileName, fileContent) {
+  try {
+    return JSON.parse(fileContent.toString());
+  } catch (err) {
+    logger.error(`Can't parse file: \`${fileName}\` as JSON. Is it properly formatted?`);
+    throw err;
+  }
+}
+var loadJson = (fileName) => {
+  const filePath = process.cwd() + `/` + fileName;
+  const fileContent = readFile(fileName, filePath);
+  return parseFileContent(fileName, fileContent);
+};
+var loadSchema = (streamId) => {
+  const schema = loadJson(`schemas/${streamId}.json`);
+  return schema;
+};
+var checkRequiredConfigKeys = (args, requiredConfigKeys) => {
+  requiredConfigKeys.forEach((configKey) => {
+    if (!args[configKey]) {
+      throw new Error(`Config is missing required key: ${configKey}`);
+    }
+  });
+};
+
+// src/sdk/service/network.ts
+var import_axios = __toESM(require("axios"));
+var import_url = require("url");
+var fs2 = __toESM(require("fs"));
+var qs = __toESM(require("qs"));
+var import_axios_retry = __toESM(require("axios-retry"));
+var shouldRetry = (error) => {
+  const statusCode = error.response ? error.response.status : 0;
+  return error.code !== "ECONNABORTED" && (!error.response || statusCode === 429 || statusCode === 409 || statusCode === 400 || statusCode >= 500 && statusCode < 599);
+};
+var getAxiosInstance = (retryCount) => {
+  const instance = import_axios.default.create({
+    timeout: 12e4
+  });
+  (0, import_axios_retry.default)(
+    instance,
+    {
+      retries: retryCount || 24,
+      retryDelay: (retryCount2, error) => {
+        logger.info(
+          `\u{1F501} Retrying for the ${retryCount2}th time the call for 
+                    method=${error.config?.method}
+                    url=${error.config?.url}
+                    params=%j
+                    body=%j`,
+          error.config?.params,
+          error.config?.data
+        );
+        if (retryCount2 > 6) {
+          return 1e4;
+        }
+        return import_axios_retry.default.exponentialDelay(retryCount2);
+      },
+      retryCondition: (error) => {
+        logger.debug(`\u{1F631} Got an error for ${error.config?.method} ${error.config?.url}
+                
+                Status: ${error?.response?.status}
+                Error code ${error.code}
+                Error config method ${error}`);
+        return (0, import_axios_retry.isNetworkOrIdempotentRequestError)(error) || shouldRetry(error);
+      }
+    }
+  );
+  return instance;
+};
+var getRawJSONApiCall = async (endpoint, config, privateLogging) => {
+  const instance = getAxiosInstance(config.retryCount);
+  const paramsSerializer = (params) => {
+    return qs.stringify(params, { arrayFormat: "repeat" });
+  };
+  try {
+    logger.info(`\u{1F916} GET ${endpoint}`, { private: privateLogging });
+    const requestConfig = {
+      params: config.qs,
+      paramsSerializer: { serialize: (params) => qs.stringify(params, { arrayFormat: "repeat" }) }
+    };
+    if (config.headers) {
+      requestConfig.headers = config.headers;
+    }
+    const response = await instance.get(
+      endpoint,
+      requestConfig
+    );
+    return response;
+  } catch (err) {
+    if (err.response.status > 400) {
+      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`. 
+
+            headers response: ${JSON.stringify(err.response.headers)}
+
+            Response body from the server: ${JSON.stringify(err.response.data)}`);
+    }
+    throw err;
+  }
+};
+var getJSONApiCallWithFullResponse = async (endpoint, config, privateLogging) => {
+  const instance = getAxiosInstance(config.retryCount);
+  const paramsSerializer = (params) => {
+    return qs.stringify(params, { arrayFormat: "repeat" });
+  };
+  try {
+    logger.info(`\u{1F916} GET ${endpoint} with params: %s`, config.qs, { private: privateLogging });
+    const requestConfig = {
+      params: config.qs,
+      paramsSerializer: { serialize: (params) => qs.stringify(params, { arrayFormat: "repeat" }) }
+    };
+    if (config.headers) {
+      requestConfig.headers = config.headers;
+    }
+    const response = await instance.get(endpoint, requestConfig);
+    return response;
+  } catch (err) {
+    if (err instanceof Error) {
+      if (import_axios.default.isAxiosError(err)) {
+        if ((err.response?.status || 0) > 400) {
+          logger.warn(`We got a \`${err.response?.status}\` status code from endpoint: \`${endpoint}\`. 
+
+            Headers response: ${JSON.stringify(err.response?.headers)}
+
+            Response body from the server: ${JSON.stringify(err.response?.data)}`);
+        }
+      }
+    }
+    throw err;
+  }
+};
+var getJSONApiCall = async (endpoint, config, privateLogging) => {
+  const fullResponse = await getJSONApiCallWithFullResponse(
+    endpoint,
+    config,
+    privateLogging
+  );
+  return fullResponse.data;
+};
+var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
+  const instance = getAxiosInstance(config.retryCount);
+  const writer = fs2.createWriteStream(output);
+  const paramsSerializer = (params) => {
+    return qs.stringify(params, { arrayFormat: "repeat" });
+  };
+  logger.info(`\u{1F916} GET ${endpoint}`, { private: privateLogging });
+  return instance({
+    method: "GET",
+    url: endpoint,
+    responseType: "stream",
+    params: config.qs,
+    ...config.headers ? { headers: config.headers } : {},
+    paramsSerializer: { serialize: (params) => qs.stringify(params, { arrayFormat: "repeat" }) }
+  }).then((response) => {
+    return new Promise((resolve, reject) => {
+      response.data.pipe(writer);
+      let error = null;
+      writer.on("error", (err) => {
+        error = err;
+        writer.close();
+        reject(err);
+      });
+      writer.on("close", () => {
+        if (!error) {
+          resolve({ outputDir: output });
+        }
+      });
+    });
+  });
+};
+var postJSONApiCall = async (endpoint, config, payload) => {
+  const instance = getAxiosInstance(config.retryCount);
+  try {
+    const requestConfig = { params: config.qs };
+    if (config.headers) {
+      requestConfig.headers = config.headers;
+    }
+    const response = await instance.post(
+      endpoint,
+      payload,
+      requestConfig
+    );
+    return response.data;
+  } catch (err) {
+    if (err?.response?.status >= 400) {
+      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`. 
+            
+            Response body from the server: ${JSON.stringify(err.response.data)}`);
+    }
+    throw err;
+  }
+};
+var postFormDataApiCall = async (endpoint, config, payload) => {
+  const instance = getAxiosInstance(config.retryCount);
+  try {
+    const response = await instance({
+      method: "post",
+      url: endpoint,
+      headers: {
+        ...payload.getHeaders()
+      },
+      data: payload
+    });
+    return response.data;
+  } catch (err) {
+    if (err.response.status > 400) {
+      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`. 
+            
+            Response body from the server: ${JSON.stringify(err.response.data)}`);
+    }
+    throw err;
+  }
+};
+var postUrlEncodedApiCall = async (endpoint, config, payload) => {
+  const instance = getAxiosInstance(config.retryCount);
+  try {
+    const params = new import_url.URLSearchParams(payload);
+    const response = await instance.post(
+      endpoint,
+      params.toString(),
+      {
+        ...config.headers ? { headers: config.headers } : {}
+      }
+    );
+    return response.data;
+  } catch (err) {
+    if (err.response.status > 400) {
+      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`. 
+            
+            Response body from the server: ${JSON.stringify(err.response.data)}`);
+    }
+    throw err;
+  }
+};
+
+// src/sdk/service/memory.ts
+var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+var printMemoryFootprint = (prefix) => {
+  const memoryData = process.memoryUsage();
+  const memoryUsage = {
+    rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+    heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+    heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+    external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`
+  };
+  logger.info(`${prefix}: %j`, memoryUsage);
+};
+
+// src/sdk/service/exit.ts
+async function gracefulExit(exitCode) {
+  if (exitCode !== 0) {
+    await loadResolver().markSyncFailed();
+  }
+  const loggerFinish = new Promise((resolve, reject) => {
+    logger.closeWinston(resolve);
+  });
+  await loggerFinish;
+  process.exit(exitCode);
+}
+
+// src/sdk/service/error.ts
+var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
+  logger.error(
+    `\u{1F4A5} Got a fatal error that will reported to the end user. Process will stop.
+        errorType=%s
+        errorText=%s
+        errorDebugText=%s
+        `,
+    errorType,
+    errorText,
+    errorDebugText
+  );
+  const resolver = loadResolver();
+  await resolver.onError(errorType, errorText, errorDebugText);
+  await gracefulExit(0);
+};
+
+// src/sdk/service/relationship.ts
+var findRelatedRelationships = (streamId, allRels) => {
+  return allRels.reduce((acc, rel) => {
+    const base = {
+      type: rel.type,
+      from: rel.from,
+      to: rel.to
+    };
+    if (rel.left === streamId) {
+      acc.right.push({
+        streamId: rel.right,
+        ...base
+      });
+    }
+    if (rel.right === streamId) {
+      acc.left.push({
+        streamId: rel.left,
+        ...base
+      });
+    }
+    return acc;
+  }, { left: [], right: [] });
+};
+
+// src/sdk/models/metadata.ts
+var MetadataMap = class extends Map {
+  areEquals(array1, array2) {
+    return array1.length === array2.length && array1.every(function(value, index) {
+      return value === array2[index];
+    });
+  }
+  findExistingEntry(key) {
+    return Array.from(super.entries()).find((entry) => {
+      if (this.areEquals(entry[0], key)) {
+        return true;
+      }
+    });
+  }
+  get(key) {
+    const existingEntry = this.findExistingEntry(key);
+    return existingEntry?.[1];
+  }
+  set(key, value) {
+    const existingKey = this.findExistingEntry(key);
+    if (existingKey) {
+      return super.set(existingKey?.[0], value);
+    } else {
+      return super.set(key, value);
+    }
+  }
+};
+var StreamMetadata = class {
+  metadataByBreadcrumb;
+  constructor() {
+    this.metadataByBreadcrumb = new MetadataMap();
+  }
+  fromCatalogStreamMetadata(metadataArr) {
+    metadataArr.forEach((metadata) => {
+      this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
+    });
+    return this;
+  }
+  toString() {
+    return this.metadataByBreadcrumb;
+  }
+  get(breadcrumb) {
+    return this.metadataByBreadcrumb.get(breadcrumb);
+  }
+  write(breadcrumb, key, value) {
+    if (value === void 0) {
+      throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
+            
+            Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
+    }
+    if (!this.metadataByBreadcrumb.get(breadcrumb)) {
+      this.metadataByBreadcrumb.set(breadcrumb, {});
+    }
+    const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
+    const newValue = {};
+    newValue[key] = value;
+    this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
+    return this;
+  }
+  getRootBreadcrumbMetadata() {
+    const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
+    if (!rootBreadcrumbMetadata) {
+      throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level. 
+            
+            Is your catalog properly generated?`);
+    }
+    return rootBreadcrumbMetadata;
+  }
+  isStreamSelected() {
+    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
+    if (rootBreadcrumbMetadata.selected === false) {
+      return false;
+    }
+    if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
+      return true;
+    }
+    return false;
+  }
+  getKeyProperties() {
+    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
+    if (!rootBreadcrumbMetadata["table-key-properties"]) {
+      return [];
+    }
+    return rootBreadcrumbMetadata["table-key-properties"];
+  }
+  getReplicationKey() {
+    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
+    if (!rootBreadcrumbMetadata["replication-key"]) {
+      return void 0;
+    }
+    return rootBreadcrumbMetadata["replication-key"];
+  }
+  getReplicationMethod() {
+    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
+    if (!rootBreadcrumbMetadata["replication-method"]) {
+      return void 0;
+    }
+    return rootBreadcrumbMetadata["replication-method"];
+  }
+  getForcedReplicationMethod() {
+    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
+    if (!rootBreadcrumbMetadata["forced-replication-method"]) {
+      return void 0;
+    }
+    return rootBreadcrumbMetadata["forced-replication-method"];
+  }
+  toList() {
+    return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
+      const metadata = this.metadataByBreadcrumb.get(breadcrumb);
+      if (!metadata) {
+        throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
+      }
+      return {
+        breadcrumb,
+        metadata
+      };
+    });
+  }
+};
+
+// src/sdk/models/catalog.ts
+var _ = __toESM(require("lodash"));
+var Catalog = class _Catalog {
+  streams;
+  static fromFile(fileName) {
+    const catalogFile = loadJson(fileName);
+    const catalog = new _Catalog();
+    catalog.streams = catalogFile.streams;
+    return catalog;
+  }
+  static fromCatalogFile(catalogFile) {
+    const catalog = new _Catalog();
+    catalog.streams = catalogFile.streams;
+    return catalog;
+  }
+  getSelectedStreams() {
+    if (!this.streams) {
+      return [];
+    }
+    const directSelectedStreams = this.streams.filter((stream) => {
+      const metadata = new StreamMetadata();
+      metadata.fromCatalogStreamMetadata(stream.metadata);
+      return metadata.isStreamSelected();
+    });
+    logger.debug(`Directly selected streams: ${directSelectedStreams.map((stream) => stream.stream).join(",")}`);
+    const getIndirectSelectedStreams = (stream, ancestorsIds) => {
+      const currAncestorsIds = ancestorsIds || [];
+      const rootBreadcrumbMetadata = stream.metadata.find((metadata) => metadata.breadcrumb.length === 0);
+      const parentStreamId = rootBreadcrumbMetadata?.metadata?.["whaly-parent-stream"];
+      if (parentStreamId) {
+        const parentStream = this.streams?.find((stream2) => stream2.stream === parentStreamId);
+        if (parentStream) {
+          currAncestorsIds.push(parentStreamId);
+          return getIndirectSelectedStreams(parentStream, currAncestorsIds);
+        }
+      }
+      return currAncestorsIds;
+    };
+    const indirectSelectedStreamIds = _.uniq(
+      directSelectedStreams.flatMap((stream) => {
+        return getIndirectSelectedStreams(stream);
+      })
+    );
+    logger.debug(`Indirectly selected streams: ${indirectSelectedStreamIds.join(",")}`);
+    const indirectSelectedStreams = indirectSelectedStreamIds.map((streamId) => {
+      if (!directSelectedStreams.find((stream) => stream.stream === streamId)) {
+        return this.streams?.find((stream) => stream.stream === streamId);
+      }
+    }).filter((stream) => !!stream);
+    return directSelectedStreams.concat(indirectSelectedStreams);
+  }
+};
+
+// src/sdk/models/messages.ts
+var RecordMessage = class {
+  type = "RECORD";
+  stream;
+  record;
+  constructor(record, stream) {
+    this.record = record;
+    this.stream = stream;
+  }
+  toString() {
+    const result = {
+      type: "RECORD",
+      record: this.record,
+      stream: this.stream
+    };
+    return JSON.stringify(result);
+  }
+};
+var StateMessage = class {
+  type = "STATE";
+  value;
+  constructor(value) {
+    this.value = value;
+  }
+  toString() {
+    const result = {
+      type: "STATE",
+      value: this.value
+    };
+    return JSON.stringify(result);
+  }
+};
+var ReplicationMethodMessage = class {
+  type = "REPLICATION_METHOD";
+  stream;
+  replication;
+  constructor(stream, version) {
+    this.stream = stream;
+    this.replication = version;
+  }
+  toString() {
+    const result = {
+      type: "REPLICATION_METHOD",
+      replication: this.replication,
+      stream: this.stream
+    };
+    return JSON.stringify(result);
+  }
+};
+var SchemaMessage = class {
+  type = "SCHEMA";
+  stream;
+  schema;
+  keyProperties;
+  bookmarkProperties;
+  displayLabel;
+  description;
+  propertiesMetadata;
+  relationships;
+  constructor(opts) {
+    const {
+      stream,
+      schema,
+      keyProperties,
+      bookmarkProperties,
+      displayLabel,
+      description,
+      relationships,
+      propertiesMetadata
+    } = opts;
+    this.stream = stream;
+    this.schema = schema;
+    this.keyProperties = keyProperties;
+    this.bookmarkProperties = bookmarkProperties;
+    this.displayLabel = displayLabel;
+    this.description = description;
+    this.relationships = relationships;
+    this.propertiesMetadata = propertiesMetadata;
+  }
+  toString() {
+    const result = {
+      type: "SCHEMA",
+      stream: this.stream,
+      schema: this.schema,
+      key_properties: this.keyProperties,
+      label: this.displayLabel,
+      description: this.description,
+      relationships: this.relationships
+    };
+    if (this.bookmarkProperties) {
+      result["bookmark_properties"] = this.bookmarkProperties;
+    }
+    return JSON.stringify(result);
+  }
+};
+
+// src/sdk/models/state.ts
+var import_moment = __toESM(require("moment"));
+
+// src/sdk/constants/date.ts
+var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
+
+// src/sdk/models/state.ts
+var import_util = require("util");
+var PROGRESS_MARKERS = "progressMarkers";
+var PROGRESS_MARKER_NOTE = "Note";
+var SIGNPOST_MARKER = "replicationKeySignpost";
+var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
+  let newRkValue = latestRecord[replicationKey];
+  if (!newRkValue) {
+    throw new Error((0, import_util.format)(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
+  }
+  let newRkValueMoment = (0, import_moment.default)(newRkValue);
+  let prevRkValue;
+  let prevRkValueMoment;
+  if (isSorted) {
+    prevRkValue = state["replicationKeyValue"];
+    prevRkValueMoment = (0, import_moment.default)(prevRkValue);
+    if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
+      throw new Error(
+        `Unsorted data detected in stream. Latest value '${newRkValue}' is
+                            smaller than previous max '${prevRkValue}'.`
+      );
+    }
+  }
+  if (!isSorted) {
+    if (!state[PROGRESS_MARKERS]) {
+      const marker = {};
+      marker[PROGRESS_MARKER_NOTE] = "Progress is not resumable if interrupted.";
+      state[PROGRESS_MARKERS] = marker;
+    }
+    prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
+    prevRkValueMoment = (0, import_moment.default)(prevRkValue || state["replicationKeyValue"]);
+    if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
+      newRkValueMoment = prevRkValueMoment;
+      newRkValue = prevRkValue;
+    }
+  }
+  const signpostMarker = state[SIGNPOST_MARKER];
+  const replicationKeySignpostMoment = (0, import_moment.default)(signpostMarker);
+  if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
+    newRkValueMoment = replicationKeySignpostMoment;
+    newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
+  }
+  if (isSorted === false) {
+    state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
+    state[PROGRESS_MARKERS]["replicationKeyValue"] = (0, import_moment.default)(newRkValueMoment).format(defaultDateTimeFormat);
+  } else {
+    state["replicationKey"] = replicationKey;
+    state["replicationKeyValue"] = (0, import_moment.default)(newRkValueMoment).format(defaultDateTimeFormat);
+  }
+  return state;
+};
+var finalizeStateProgressMarkers = (state) => {
+  const signpostValue = state[SIGNPOST_MARKER];
+  const progressMarkers = state[PROGRESS_MARKERS];
+  if (progressMarkers && progressMarkers["replicationKey"] && progressMarkers["replicationKeyValue"]) {
+    state["replicationKey"] = progressMarkers["replicationKey"];
+    let newRkValue = progressMarkers["replicationKeyValue"];
+    if (signpostValue && _greaterThan(newRkValue, signpostValue)) {
+      newRkValue = signpostValue;
+    }
+    const previousReplicationValue = state["replicationKeyValue"];
+    if (!previousReplicationValue || previousReplicationValue && _greaterThan(newRkValue, previousReplicationValue)) {
+      state["replicationKeyValue"] = newRkValue;
+    }
+  }
+  delete state[PROGRESS_MARKERS];
+  return state;
+};
+var _greaterThan = (aValue, bValue) => {
+  return (0, import_moment.default)(aValue).isAfter((0, import_moment.default)(bValue));
+};
+var extractStateForStream = (state, streamId) => {
+  if (!state.bookmarks) {
+    state.bookmarks = {};
+  }
+  if (!state.bookmarks[streamId]) {
+    state.bookmarks[streamId] = {};
+  }
+  return state.bookmarks[streamId];
+};
+var StateService = class {
+  static instance;
+  bookmarks;
+  constructor() {
+    this.bookmarks = {};
+  }
+  static getInstance() {
+    if (!this.instance) {
+      this.instance = new this();
+    }
+    return this.instance;
+  }
+  // To be deprecated
+  setBookmark(streamId, ts) {
+    this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
+  }
+  setBookmarkV2(streamId, streamState) {
+    this.bookmarks[streamId] = streamState;
+  }
+  setBookmarkSignpostV2(streamId, signpostValue) {
+    if (!this.bookmarks[streamId]) {
+      this.bookmarks[streamId] = {};
+    }
+    this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
+  }
+  getBookmark(streamId) {
+    if (!this.bookmarks[streamId]) {
+      return (0, import_moment.default)(0);
+    }
+    return (0, import_moment.default)(this.bookmarks[streamId]);
+  }
+  getBookmarkV2(streamId) {
+    if (this.bookmarks[streamId]) {
+      return this.bookmarks[streamId];
+    } else {
+      return {};
+    }
+  }
+  clearState() {
+    this.bookmarks = {};
+  }
+  get() {
+    const state = {
+      bookmarks: this.bookmarks
+    };
+    return state;
+  }
+  async forwardStateToTarget(target) {
+    const state = {
+      bookmarks: this.bookmarks
+    };
+    const message = new StateMessage(state);
+    await target.state(message);
+  }
+};
+
+// src/sdk/models/tap/authenticator.ts
+var Authenticator = class {
+  getAuthHeaders(config) {
+    return Promise.resolve({});
+  }
+  getAuthQS(config) {
+    return Promise.resolve({});
+  }
+};
+
+// src/sdk/models/tap/restStream.ts
+var import_axios2 = __toESM(require("axios"));
+var qs2 = __toESM(require("qs"));
+
+// src/sdk/models/tap/stream.ts
+var import_moment2 = __toESM(require("moment"));
+
+// src/sdk/helpers/typing.ts
+var isDatetimeType = (typeDef) => {
+  if (!typeDef) {
+    throw new Error("Could not detect type from empty typeDef param.");
+  }
+  if (typeDef["anyOf"] !== void 0) {
+    for (const subTypeDef of typeDef["anyOf"]) {
+      if (isDatetimeType(subTypeDef)) {
+        return true;
+      }
+    }
+    return false;
+  } else if (typeDef["type"] !== void 0) {
+    return typeDef["format"] === "date-time";
+  }
+  throw new Error(
+    `Could not detect type of key using schema '${typeDef}'`
+  );
+};
+
+// src/sdk/models/tap/stream.ts
+var _2 = __toESM(require("lodash"));
+var import_bluebird2 = require("bluebird");
+
+// src/sdk/models/tap/tap.ts
+var import_bluebird = require("bluebird");
+var import_dependency_graph = require("dependency-graph");
+var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
+var Tap = class {
+  _target;
+  _config;
+  _streams;
+  _concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
+  constructor(target, config) {
+    this._streams = [];
+    this._target = target;
+    this._config = config;
+  }
+  discover = async (existingCatalog) => {
+    logger.debug("Loading schemas");
+    const streamDepGraph = new import_dependency_graph.DepGraph();
+    const buildDependencyGraph = (streams2, parentId) => {
+      streams2.forEach((s) => {
+        streamDepGraph.addNode(s.streamId, s);
+        if (parentId) {
+          streamDepGraph.addDependency(s.streamId, parentId);
+        }
+        if (s.children) {
+          return buildDependencyGraph(s.children, s.streamId);
+        }
+      });
+    };
+    buildDependencyGraph(this._streams);
+    const discoveredStreamCatalogEntries = await import_bluebird.Promise.map(
+      streamDepGraph.overallOrder(),
+      async (streamId) => {
+        logger.info(`\u2728 Loading schema for ${streamId}`);
+        const stream = streamDepGraph.getNodeData(streamId);
+        const schema = await stream.getSchema();
+        const parentStreamIds = streamDepGraph.directDependenciesOf(streamId);
+        if (parentStreamIds.length > 1) {
+          throw new Error(`StreamId=${streamId} directly depends on more than 1 stream, e.g. ${parentStreamIds.join(",")} which is not supported!`);
+        }
+        const parentStreamId = parentStreamIds?.[0];
+        const { metadata } = await this.generateMetadataFromStream(stream, parentStreamId);
+        return {
+          schema,
+          metadata,
+          stream: stream.streamId,
+          tap_stream_id: stream.streamId
+        };
+      },
+      { concurrency: this._concurrency }
+    );
+    const existingCatalogStreamIds = (existingCatalog?.streams || []).map((entry) => {
+      return entry.stream;
+    });
+    const newlyDiscoveredStreamCatalogEntries = discoveredStreamCatalogEntries.filter((entry) => {
+      if (existingCatalogStreamIds.includes(entry.stream)) {
+        return false;
+      } else {
+        return true;
+      }
+    });
+    logger.debug(`Newly discovered streams: %j`, newlyDiscoveredStreamCatalogEntries);
+    const allDiscoveredStreamIds = discoveredStreamCatalogEntries.map((entry) => {
+      return entry.stream;
+    });
+    const existingCatalogMinusDeletedStreams = (existingCatalog?.streams || []).filter((entry) => {
+      if (!allDiscoveredStreamIds.includes(entry.stream)) {
+        logger.debug(`Stream %s was deleted, removing it from the catalog`, entry.stream);
+        return false;
+      } else {
+        return true;
+      }
+    });
+    const updatedExistingCatalog = existingCatalogMinusDeletedStreams.map((existingCatalogEntry) => {
+      const matchingDiscoveredCatalogEntry = discoveredStreamCatalogEntries.find((entry) => entry.stream === existingCatalogEntry.stream);
+      if (matchingDiscoveredCatalogEntry) {
+        const existingRootMetadata = existingCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
+        const discoveredRootMetadata = matchingDiscoveredCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
+        if (existingRootMetadata) {
+          const parentStreamVal = discoveredRootMetadata?.metadata["whaly-parent-stream"];
+          if (parentStreamVal !== void 0) {
+            existingRootMetadata.metadata["whaly-parent-stream"] = parentStreamVal;
+          }
+          const forcedReplicationMethodVal = discoveredRootMetadata?.metadata["forced-replication-method"];
+          if (forcedReplicationMethodVal !== void 0) {
+            existingRootMetadata.metadata["forced-replication-method"] = forcedReplicationMethodVal;
+          }
+          const forcedReplicationKeyVal = discoveredRootMetadata?.metadata["forced-replication-key"];
+          if (forcedReplicationKeyVal !== void 0) {
+            existingRootMetadata.metadata["forced-replication-key"] = forcedReplicationKeyVal;
+          }
+          const validReplicationKeysVal = discoveredRootMetadata?.metadata["valid-replication-keys"];
+          if (validReplicationKeysVal !== void 0) {
+            existingRootMetadata.metadata["valid-replication-keys"] = validReplicationKeysVal;
+          }
+          const rowCountVal = discoveredRootMetadata?.metadata["row-count"];
+          if (rowCountVal !== void 0) {
+            existingRootMetadata.metadata["row-count"] = rowCountVal;
+          }
+        }
+      }
+      return existingCatalogEntry;
+    });
+    const streams = updatedExistingCatalog.concat(newlyDiscoveredStreamCatalogEntries);
+    return { streams };
+  };
+  sync = async (catalog) => {
+    await this.initiateSync(catalog);
+    logger.debug(`[TAP] All streams have finished syncing. We send the [complete] signal to the target.`);
+    await this._target.complete();
+    return Promise.resolve();
+  };
+  async generateMetadataFromStream(stream, parentStreamId) {
+    const metadata = new StreamMetadata();
+    metadata.write([], "table-key-properties", stream.primaryKey);
+    const defaultReplicationConfig = stream.defaultReplicationConfig();
+    metadata.write([], "replication-method", defaultReplicationConfig.replicationMethod);
+    const validReplicationkeys = await stream.getValidReplicationKeys();
+    metadata.write([], "valid-replication-keys", validReplicationkeys);
+    if (defaultReplicationConfig.isForced) {
+      metadata.write([], "forced-replication-method", defaultReplicationConfig.replicationMethod);
+      if (stream.forcedReplicationKey) {
+        metadata.write([], "forced-replication-key", stream.forcedReplicationKey);
+      }
+    }
+    const rowCount = await stream.getRowCount();
+    if (rowCount !== void 0) {
+      metadata.write([], "row-count", rowCount);
+    }
+    if (stream.isSilent === true) {
+      metadata.write([], "whaly-is-hidden", stream.isSilent);
+    }
+    if (stream.isSilent === false) {
+      metadata.write([], "selected-by-default", stream.selectedByDefault);
+    }
+    if (stream.displayLabel) {
+      metadata.write([], "whaly-display-label", stream.displayLabel);
+    }
+    if (stream.description) {
+      metadata.write([], "whaly-description", stream.description);
+    }
+    if (parentStreamId !== void 0) {
+      metadata.write([], "whaly-parent-stream", parentStreamId);
+    }
+    return { metadata: metadata.toList() };
+  }
+  async initiateSync(catalog) {
+    logger.info(`\u{1F680} Start syncing`);
+    const state = StateService.getInstance().get();
+    logger.info(`\u{1F4CD} Received state: %j`, state);
+    logger.debug(`\u{1F4DA} Received catalog: %j`, catalog);
+    const selectedStreams = catalog.getSelectedStreams();
+    if (selectedStreams.length === 0) {
+      logger.info(`\u{1F631} There is no stream to sync. 
+            
+            Did you update the \`catalog.json\` to add \`"selected": true\` in the table metadata associated with the root breadcrumb (e.g. \`[]\`)`);
+      return;
+    }
+    const selectedStreamIds = selectedStreams.map((stream) => stream.stream);
+    logger.info(`\u{1F973} Syncing selected streams: %s`, selectedStreamIds.join(","));
+    await import_bluebird.Promise.map(
+      this._streams,
+      (stream) => {
+        if (selectedStreamIds.includes(stream.streamId)) {
+          const selectedStream = selectedStreams.find(((selectedStream2) => selectedStream2.stream === stream.streamId));
+          const rootMetadataEntry = selectedStream?.metadata.find((streamMdt) => streamMdt.breadcrumb.length === 0);
+          if (!rootMetadataEntry) {
+            throw new Error(`Can't find a catalog root metadata entry for StreamId=${stream.streamId}`);
+          }
+          stream.setRootMetadataEntry(rootMetadataEntry.metadata);
+          return stream.sync();
+        }
+      },
+      {
+        concurrency: this._concurrency
+      }
+    );
+  }
+  // Used to clean up any ressource before exit
+  end() {
+    return Promise.resolve();
+  }
+};
+
+// src/sdk/models/tap/stream.ts
+var import_util2 = require("util");
+var StreamV2 = class {
+  // Runtime values, can't be overriden
+  _config;
+  _tapName = "default";
+  _tapState;
+  _target;
+  // Contains the root metadata entry of the catalog stream configuration
+  // contains some end user schema configuration
+  rootMetadataEntry;
+  // Compile time values, can be overriden
+  // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
+  forcedReplicationMethod = void 0;
+  selectedByDefault = true;
+  STATE_MSG_FREQUENCY = 10;
+  streamId = "default";
+  schemaPath = void 0;
+  displayLabel = void 0;
+  description = void 0;
+  primaryKey = [];
+  forcedReplicationKey = void 0;
+  // When set, use the state from another stream for this stream
+  useStateFromStreamId = void 0;
+  relationships = [];
+  children = [];
+  // If true, it means that the records in this stream are sorted by replicationKey.
+  // If false, then the records are coming in an unsorted manner.
+  isSorted = false;
+  // If true, then no SCHEMA / RECORD messages will be sent for this Stream.
+  // However, a STATE will still be emitted if the Stream is using an incremental sync type.
+  isSilent = false;
+  childConcurrency;
+  // Metrics configuration
+  rowsSyncedMetricsConf;
+  executionTimeMetricsConf;
+  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    this._tapName = tapName;
+    this._config = config;
+    this._tapState = _2.cloneDeep(state);
+    this._target = target;
+    this.childConcurrency = childConcurrency;
+    this.rowsSyncedMetricsConf = {
+      isEnabled: () => !this.isSilent,
+      getStreamIds: () => [this.streamId]
+    };
+    this.executionTimeMetricsConf = {
+      isEnabled: () => !this.isSilent,
+      getStreamIds: () => [this.streamId]
+    };
+  }
+  setRootMetadataEntry = (rootMetadataEntry) => {
+    this.rootMetadataEntry = rootMetadataEntry;
+  };
+  /**
+   * Sync this stream
+   * Called only for streams with no parents, aka. "root tap streams"
+   * @returns 
+   */
+  sync = async () => {
+    try {
+      const replicationMethod = this.configuredReplicationConfig().replicationMethod;
+      logger.info(
+        `\u{1F3AC} Beginning ${replicationMethod} sync of stream '${this.streamId}'...`
+      );
+      await this.asyncInit();
+      await this._writeSchemaMessage();
+      await this._writeReplicationMethodMessage();
+      await this._writeSignpostInState();
+      await this._syncRecords();
+      return this.flush();
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error((0, import_util2.format)(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
+      }
+      throw err;
+    }
+  };
+  flush = async () => {
+    const replicationMethod = this.configuredReplicationConfig().replicationMethod;
+    logger.info(
+      `\u{1F6BD} Flushing ${replicationMethod} sync of stream '${this.streamId}' + all of its children...`
+    );
+    await this._flush();
+    await import_bluebird2.Promise.map(this.children, (child) => {
+      return child.flush();
+    }, { concurrency: 3 });
+  };
+  // Private message authoring methods:
+  /**
+   * Write out a STATE message with the latest state.
+   */
+  _writeStateMessage = () => {
+    const message = new StateMessage(StateService.getInstance().get());
+    return this._target.state(message);
+  };
+  /**
+   * Write out a ACTIVATE_VERSION message.
+   */
+  _writeReplicationMethodMessage = async () => {
+    const method = this.configuredReplicationConfig().replicationMethod;
+    const message = new ReplicationMethodMessage(this.streamId, method);
+    await import_bluebird2.Promise.map(this.children, async (c) => {
+      return c._writeReplicationMethodMessage();
+    }, { concurrency: 3 });
+    await this._target.replicationMethod(message);
+  };
+  /**
+   * Write out a SCHEMA message with the stream schema.
+   */
+  _writeSchemaMessage = async (skipChildrenSchema) => {
+    if (!this.isSilent) {
+      const schema = await this.getSchema();
+      if (schema !== void 0) {
+        const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
+        const replicationConfig = this.configuredReplicationConfig();
+        const message = new SchemaMessage({
+          keyProperties: this.primaryKey,
+          stream: this.streamId,
+          schema: schema.jsonSchema,
+          relationships: relatedRels,
+          ...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
+          ...this.displayLabel ? { displayLabel: this.displayLabel } : {},
+          ...this.description ? { description: this.description } : {},
+          ...schema.propertiesMetadata ? { propertiesMetadata: schema.propertiesMetadata } : {}
+        });
+        await this._target.schema(message);
+      }
+    }
+    if (skipChildrenSchema !== true) {
+      await import_bluebird2.Promise.map(this.children, async (child) => {
+        await child._writeSchemaMessage();
+      }, { concurrency: 3 });
+    }
+  };
+  // Private bookmarking methods
+  _writeSignpostInState = async () => {
+    const signpostMoment = await this.getReplicationKeySignpost();
+    const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
+    if (signpostValue) {
+      StateService.getInstance().setBookmarkSignpostV2(this.streamId, signpostValue);
+    }
+  };
+  /**
+   * Update state of stream with data from the provided record. 
+   */
+  _incrementStreamState = (latestRecord) => {
+    if (latestRecord) {
+      const replicationConfig = this.configuredReplicationConfig();
+      if ([
+        "INCREMENTAL",
+        "LOG_BASED"
+      ].includes(replicationConfig.replicationMethod)) {
+        if (this.useStateFromStreamId) {
+          return;
+        }
+        if (this.forcedReplicationMethod && !this.forcedReplicationKey) {
+          return;
+        }
+        if (!replicationConfig.replicationKey) {
+          throw new Error(
+            `Could not detect replication key for '${this.streamId}' stream"
+                        replication method=${replicationConfig.replicationMethod})`
+          );
+        }
+        const stateServiceInst = StateService.getInstance();
+        const state = stateServiceInst.getBookmarkV2(this.streamId);
+        const newState = incrementStreamState(
+          state,
+          replicationConfig.replicationKey,
+          latestRecord,
+          this.isSorted
+        );
+        StateService.getInstance().setBookmarkV2(this.streamId, newState);
+      }
+    }
+  };
+  /**
+   * Return True if the stream uses a timestamp-based replication key.
+   * 
+   * Developers can override with `is_timestamp_replication_key = True` in
+      order to force this value.
+   */
+  isTimestampReplicationKey = async () => {
+    const replicationConfig = this.configuredReplicationConfig();
+    if (!replicationConfig.replicationKey) {
+      return false;
+    }
+    const schema = await this.getSchema();
+    if (!schema || !schema.jsonSchema || !schema.jsonSchema.properties) {
+      return false;
+    }
+    const typeFromSchema = schema.jsonSchema["properties"][replicationConfig.replicationKey];
+    if (!typeFromSchema) {
+      return false;
+    }
+    return isDatetimeType(typeFromSchema);
+  };
+  /**
+   * Return state timestamp if set for the stream, 
+   * or `start_date` config if set, 
+   * or the UNIX Epoch
+   **/
+  getStartingTimestamp() {
+    const streamIdToReadFrom = this.useStateFromStreamId ? this.useStateFromStreamId : this.streamId;
+    const state = extractStateForStream(this._tapState, streamIdToReadFrom);
+    const startDate = this._config.start_date;
+    const replicationConfig = this.configuredReplicationConfig();
+    logger.debug(`${this.streamId}: state.replicationKeyValue: ${state.replicationKeyValue}, 
+        state.replicationKey: ${state.replicationKey},
+        this.replicationKey: ${replicationConfig.replicationKey}`);
+    if (state.replicationKeyValue) {
+      logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
+      return (0, import_moment2.default)(state.replicationKeyValue);
+    } else if (startDate) {
+      logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
+      return (0, import_moment2.default)(startDate);
+    } else {
+      logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
+      return (0, import_moment2.default)(0);
+    }
+  }
+  /**
+   * Return the max allowable bookmark value for this stream's replication key.
+   * 
+   * For timestamp-based replication keys, this defaults to `moment()`. For
+   * non-timestamp replication keys, default to `undefined`.
+   * 
+   * Override this value to prevent bookmarks from being advanced in cases where we
+   * may only have a partial set of records.
+   */
+  getReplicationKeySignpost = async () => {
+    if (await this.isTimestampReplicationKey()) {
+      return (0, import_moment2.default)();
+    }
+    return void 0;
+  };
+  /**
+   * Return the default replication method for the stream that will be used to write the catalog.
+   */
+  defaultReplicationConfig = () => {
+    if (this.forcedReplicationMethod) {
+      return {
+        replicationMethod: this.forcedReplicationMethod,
+        replicationKey: this.forcedReplicationKey,
+        isForced: true
+      };
+    }
+    if (this.forcedReplicationKey || this.useStateFromStreamId) {
+      return {
+        replicationMethod: "INCREMENTAL",
+        replicationKey: this.forcedReplicationKey,
+        isForced: true
+      };
+    }
+    return {
+      replicationMethod: "FULL_TABLE",
+      replicationKey: void 0,
+      isForced: false
+    };
+  };
+  /**
+   * Return the configured replication method that will be used for the sync.
+   */
+  configuredReplicationConfig = () => {
+    const defaultReplicationMethod = this.defaultReplicationConfig();
+    if (defaultReplicationMethod.isForced) {
+      return {
+        replicationMethod: defaultReplicationMethod.replicationMethod,
+        replicationKey: defaultReplicationMethod.replicationKey
+      };
+    }
+    return {
+      replicationMethod: this.rootMetadataEntry?.["replication-method"] || defaultReplicationMethod.replicationMethod,
+      replicationKey: this.rootMetadataEntry?.["replication-key"] || defaultReplicationMethod.replicationKey
+    };
+  };
+  // Private sync methods:
+  async _syncRecords(parent) {
+    let rowsSent = 0;
+    let recordSyncedMetrics = [];
+    if (this.rowsSyncedMetricsConf.isEnabled() === true) {
+      recordSyncedMetrics = getCounterMetrics(
+        ROWS_SYNCED_METRIC_NAME,
+        this.rowsSyncedMetricsConf.getStreamIds ? this.rowsSyncedMetricsConf.getStreamIds() : []
+      );
+    }
+    let executionTimeMetrics = [];
+    if (this.executionTimeMetricsConf.isEnabled() === true) {
+      executionTimeMetrics = getCounterMetrics(
+        EXECUTION_TIME_METRIC_NAME,
+        this.executionTimeMetricsConf.getStreamIds ? this.executionTimeMetricsConf.getStreamIds() : []
+      );
+    }
+    const startTime = (/* @__PURE__ */ new Date()).getTime();
+    for await (const row of this._getRecords(parent)) {
+      if ((rowsSent - 1) % this.STATE_MSG_FREQUENCY == 0) {
+        await this._writeStateMessage();
+      }
+      const recordMessage = new RecordMessage(
+        { ...row },
+        this.streamId
+      );
+      if (!this.isSilent) {
+        await this._target.record(recordMessage);
+      }
+      await import_bluebird2.Promise.map(
+        this.children,
+        async (child) => {
+          await child.asyncInit(row);
+          return child._syncRecords(row);
+        },
+        {
+          concurrency: this.childConcurrency
+        }
+      );
+      this._incrementStreamState(row);
+      recordSyncedMetrics.forEach((metric) => {
+        metric.increment();
+      });
+      rowsSent += 1;
+    }
+    const stateServiceInst = StateService.getInstance();
+    const state = stateServiceInst.getBookmarkV2(this.streamId);
+    const newState = finalizeStateProgressMarkers(state);
+    stateServiceInst.setBookmarkV2(this.streamId, newState);
+    if (!parent) {
+      logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
+    } else {
+      logger.debug(`\u2705 Completed sync for sub-stream: ${this.streamId} (${rowsSent} records)`);
+    }
+    await this._writeStateMessage();
+    const endTime = (/* @__PURE__ */ new Date()).getTime();
+    const executionTimeMs = endTime - startTime;
+    executionTimeMetrics.forEach((metric) => {
+      metric.increment(executionTimeMs);
+    });
+  }
+  // Overridable Methods
+  asyncInit(parent) {
+    return Promise.resolve();
+  }
+  /**
+   * Return the stream schema.
+   * 
+   * Can be overriden to return a dynamic schema
+   */
+  getSchema() {
+    logger.debug(`stream: ${this.streamId} - getSchema() is called`);
+    if (this.schemaPath) {
+      const schema = loadJson(`schemas/${this._tapName}/${this.schemaPath}`);
+      return Promise.resolve({ jsonSchema: schema });
+    } else {
+      if (this.isSilent) {
+        return Promise.resolve(void 0);
+      }
+      throw new Error(`Stream=${this.streamId} - No schema path was passed.
+                    Either you forgot to fill the the \`schemaPath\` property,
+                    or to override the getSchema() method.`);
+    }
+  }
+  getRowCount() {
+    return Promise.resolve(void 0);
+  }
+  /**
+   * Abstract row generator function. Must be overridden by the child class.
+   */
+  _getRecords(parent) {
+    throw new Error(`Stream._getRecords() - Should be implemented by child class`);
+  }
+  /**
+   * Return the list of keys that can be configured in the UI by the end user as the replication key
+   * @returns 
+   */
+  getValidReplicationKeys() {
+    return Promise.resolve([]);
+  }
+  /**
+   * Called at the end of the sync. Use to flush all records before closing the Stream.
+   * @returns 
+   */
+  async _flush() {
+    await Promise.resolve();
+  }
+};
+
+// src/sdk/helpers/qs-logger.ts
+var _3 = __toESM(require("lodash"));
+var MAX_QS_KEYS = 10;
+var MAX_QS_STRING_LENGTH = 200;
+var MAX_QS_ARRAY_ITEMS = 5;
+var MAX_QS_DEPTH = 2;
+function truncateQueryParams(value, depth = 0) {
+  if (value == null) return value;
+  if (depth >= MAX_QS_DEPTH) return "[\u2026]";
+  if (typeof value === "string") {
+    if (value.length <= MAX_QS_STRING_LENGTH) return value;
+    const truncated = value.slice(0, MAX_QS_STRING_LENGTH);
+    return `${truncated}\u2026(+${value.length - MAX_QS_STRING_LENGTH} chars)`;
+  }
+  if (Array.isArray(value)) {
+    const items = value.slice(0, MAX_QS_ARRAY_ITEMS).map((v) => truncateQueryParams(v, depth + 1));
+    const omittedCount = Math.max(0, value.length - MAX_QS_ARRAY_ITEMS);
+    if (omittedCount > 0) {
+      items.push(`[\u2026 ${omittedCount} more items]`);
+    }
+    return items;
+  }
+  if (_3.isPlainObject(value)) {
+    const keys = Object.keys(value);
+    const result = {};
+    for (const key of keys.slice(0, MAX_QS_KEYS)) {
+      result[key] = truncateQueryParams(value[key], depth + 1);
+    }
+    const omittedCount = Math.max(0, keys.length - MAX_QS_KEYS);
+    if (omittedCount > 0) {
+      result["__omittedKeys"] = omittedCount;
+    }
+    return result;
+  }
+  return value;
+}
+function getTruncatedParamsForLog(params) {
+  try {
+    return truncateQueryParams(params);
+  } catch {
+    return "[unserializable params]";
+  }
+}
+
+// src/sdk/models/tap/restStream.ts
+var _4 = __toESM(require("lodash"));
+var RESTStreamV2 = class extends StreamV2 {
+  axiosInstance;
+  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    super(tapName, config, state, target, childConcurrency);
+    this.apiCallsMetricsConf = {
+      isEnabled: () => !this.isSilent,
+      getStreamIds: () => [this.streamId]
+    };
+  }
+  // Internal
+  // Those functions shouldn't be overriden (or it means that we fuck'ed up the SDK during the design phase)
+  /**
+   * Prepare an Axios request object.
+   * Pagination information can be parsed from `nextPageToken` if `nextPageToken` is not undefined.
+   * @param nextPageToken 
+   * @returns 
+   */
+  _prepareRequest = async (nextPageToken, parent) => {
+    const method = this.httpMethod;
+    const url = this.getUrl(parent);
+    if (!url) {
+      throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
+    }
+    const authenticator = this.authenticator;
+    const params = {
+      ...this.getNextUrlParams(nextPageToken, parent),
+      ...await authenticator?.getAuthQS(this._config)
+    };
+    const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
+    let headers = this.getCustomHTTPHeaders(parent);
+    if (!headers) {
+      headers = {};
+    }
+    if (authenticator) {
+      const authHeaders = await authenticator.getAuthHeaders(this._config);
+      headers = {
+        ...headers,
+        ...authHeaders
+      };
+    }
+    const paramsSerializer = (params2) => {
+      return qs2.stringify(params2, { arrayFormat: "repeat" });
+    };
+    const request = {
+      url,
+      method,
+      data: requestBody,
+      params,
+      paramsSerializer,
+      headers
+    };
+    return request;
+  };
+  async _requestWithRetry(request, privateLogging) {
+    try {
+      logger.debug(
+        `\u{1F916} ${request.method} ${request.url} with params: %j | body: %j`,
+        request.params,
+        request.data,
+        { private: privateLogging }
+      );
+      logger.info(
+        `\u{1F916} ${request.method} ${request.url} qs: %j`,
+        getTruncatedParamsForLog(request.params),
+        { private: privateLogging }
+      );
+      const axiosInstance = this.axiosInstance;
+      if (!axiosInstance) {
+        throw new Error(`Axios Instance is not initialized!`);
+      }
+      const response = await axiosInstance.request(request);
+      return response.data;
+    } catch (err) {
+      if (import_axios2.default.isAxiosError(err)) {
+        logger.error(
+          `\u{1F916} Error on ${request.method} ${request.url} with params %j | body: %j | headers: %j - getting error %s - response body: %j`,
+          request.params,
+          request.data,
+          request.headers,
+          err.message,
+          err.response?.data
+        );
+        const recoveredValue = await this.httpErrorHandler(request.url, err);
+        if (recoveredValue === void 0) {
+          return "STOP_SYNC";
+        } else {
+          return recoveredValue;
+        }
+      } else {
+        throw err;
+      }
+    }
+  }
+  async *_requestRecords(parent) {
+    let nextPageToken = void 0;
+    let finished = false;
+    let apiCallsMetrics = [];
+    if (this.apiCallsMetricsConf.isEnabled() === true) {
+      apiCallsMetrics = getCounterMetrics(
+        API_CALLS_METRIC_NAME,
+        this.apiCallsMetricsConf.getStreamIds ? this.apiCallsMetricsConf.getStreamIds() : []
+      );
+    }
+    this.axiosInstance = getAxiosInstance(this.httpRetryCount);
+    while (!finished) {
+      const preparedRequest = await this._prepareRequest(nextPageToken, parent);
+      const resp = await this._requestWithRetry(preparedRequest);
+      apiCallsMetrics.forEach((metric) => {
+        metric.increment();
+      });
+      if (resp === "STOP_SYNC") {
+        return;
+      }
+      const rows = this.parseResponse(resp, parent, nextPageToken);
+      for await (const row of rows) {
+        yield row;
+      }
+      const previousToken = _4.cloneDeep(nextPageToken);
+      nextPageToken = this.getNextPageToken(resp, previousToken);
+      if (nextPageToken && _4.isEqual(nextPageToken, previousToken)) {
+        throw new Error(
+          `${this.streamId} - Loop detected in pagination. "
+                    Pagination token: \`${JSON.stringify(nextPageToken)}\` is identical to prior token \`${JSON.stringify(previousToken)}\`.`
+        );
+      }
+      finished = !nextPageToken;
+    }
+  }
+  async *_getRecords(parent) {
+    for await (const row of this._requestRecords(parent)) {
+      yield row;
+    }
+  }
+  // Overridable
+  // Even if the default impl. of some function can be used as-is, those functions
+  // are meant to be overriden when necessary to match the API needs.
+  httpMethod = "GET";
+  baseUrl = "";
+  path = "";
+  httpRetryCount = 24;
+  authenticator;
+  apiCallsMetricsConf;
+  // When undefined is returned, we should stop the sync without any error
+  async httpErrorHandler(url, err) {
+    if ((err.response?.status || 0) > 400) {
+      throw new Error(`We got a \`${err.response?.status}\` status code from endpoint: \`${url}\`. 
+
+            Response body from the server: ${JSON.stringify(err.response?.data)}`);
+    }
+    throw err;
+  }
+  /** 
+   * Return a dictionary of values to be used in URL parameterization.
+   * If paging is supported, developers may override with specific paging logic.
+   * 
+   * By default, no params are passed.
+  */
+  getNextUrlParams(nextPageToken, parent) {
+    return {};
+  }
+  /** 
+   * Prepare the data payload for the REST API request.
+   * Useful when working with POST APIs.
+   * 
+   * By default, no payload will be sent (return undefined).
+  */
+  getRequestBodyForNextCall(nextPageToken, parent) {
+    return void 0;
+  }
+  /**
+   * Return token identifying next page or undefined if all records have been read.
+   * 
+   * By default, no token will be returned and a single API call will be made.
+   * 
+   * @param response 
+   * @param previousToken 
+   * @returns 
+   */
+  getNextPageToken(response, previousToken) {
+    return void 0;
+  }
+  /**
+   * Return custom headers to be used for HTTP requests.
+   * 
+   * By default, no additional header will be added
+   */
+  getCustomHTTPHeaders(parent) {
+    return void 0;
+  }
+  /**
+   * Return a URL, optionally targeted to a specific partition.
+   * This can be overriden to perform dynamic URL generation.
+   * 
+   * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
+   * @returns 
+   */
+  getUrl(parent) {
+    const urlPattern = [this.baseUrl, this.path].join("");
+    return urlPattern;
+  }
+  /**
+   * Parse the response and return an iterator of result rows.
+   * 
+   * Default impl assume that R = O or R = O[]
+   * @param response
+   */
+  async *parseResponse(response, parent, nextPageToken) {
+    if (Array.isArray(response)) {
+      for (const row of response) {
+        yield row;
+      }
+    } else {
+      yield response;
+    }
+  }
+};
+
+// src/sdk/models/target/dbSync.ts
+var import_async_retry = __toESM(require("async-retry"));
+var StreamWarehouseSyncService = class {
+  streamId;
+  database;
+  schema;
+  table;
+  flattenedSchema;
+  primaryKeys;
+  maxRetryCount = 5;
+  renamedColumnStore;
+  constructor(streamId, database, schema, table, renameColumnStore) {
+    this.streamId = streamId;
+    this.database = database;
+    this.schema = schema;
+    this.table = table;
+    this.primaryKeys = [];
+    this.flattenedSchema = {};
+    this.renamedColumnStore = renameColumnStore;
+  }
+  getcolumnNameWithTypeSuffix(colName, warehouseType) {
+    return `${colName}__${warehouseType}`;
+  }
+  // Orchestration functions
+  updateSchemaInWarehouse = async (flattenedSchema, primaryKeys) => {
+    await this.createDatabaseAndSchemaIfNotExists(0);
+    this.flattenedSchema = flattenedSchema;
+    this.renamedColumnStore.computeColumnNameForStream(
+      this.streamId,
+      flattenedSchema
+    );
+    this.primaryKeys = primaryKeys.map((pk) => {
+      return this.renamedColumnStore.getColumnTranslation(this.streamId, pk);
+    });
+    await this.syncTableSchemaWithWarehouse();
+  };
+  renameColumns(record) {
+    const renamedColsRecord = {};
+    Object.keys(record).forEach((originalColName) => {
+      renamedColsRecord[this.renamedColumnStore.getColumnTranslation(this.streamId, originalColName)] = record[originalColName];
+    });
+    return renamedColsRecord;
+  }
+  // Function to execute queries on the warehouse
+  runQueriesWithRetry = async (queries) => {
+    await (0, import_async_retry.default)(
+      async () => {
+        await this.runQueries(queries);
+      },
+      {
+        retries: 5
+      }
+    );
+  };
+  syncTableSchemaWithWarehouse = async () => {
+    const tables = await this.getTablesInSchema();
+    const foundTables = tables.filter((table) => this.table === table.tableName);
+    if (foundTables.length === 0) {
+      logger.info(`\u{1F423} Database: \`${this.database}\`, Schema: \`${this.schema}\`, Table: \`${this.table}\` does not exist in Warehouse. Creating it...`);
+      await this.createTable();
+    } else if (foundTables.length === 1) {
+      logger.info(`\u{1F44B} Database: \`${this.database}\`, Schema: \`${this.schema}\`, Table: \`${this.table}\` already exists. Updating the columns if needed.`);
+      await this.updateTableColumns();
+    }
+  };
+  /**
+   * When there is a conflict between the synced data type and the data type of an
+   * already existing column in the Warehouse, this create a new column in Warehouse with the new
+   * datatype as suffix.
+   * 
+   * This refer the fact that data should now be synced with the new column.
+   * 
+   * @param tableFields 
+   */
+  addVersionedColumns = async (tableFields) => {
+    if (tableFields.length > 0) {
+      logger.info(`\u{1F6C2} Stream: ${this.streamId} - Versionning following columns in Warehouse schema: ${JSON.stringify(tableFields)}`);
+      const columnsFromWarehouse = await this.getTableColumnsFromWarehouse();
+      const columnNamesFromWarehouse = columnsFromWarehouse.map((col) => col.name);
+      let columnsToBeAddedInWarehouse = [];
+      tableFields.forEach((tblField) => {
+        const safeName = tblField.name;
+        const warehouseTypeToBeUsed = this.getWarehouseTypeFromJSONSchema(tblField.definition);
+        const columnNameWithTypeSuffix = this.getcolumnNameWithTypeSuffix(safeName, warehouseTypeToBeUsed);
+        this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId)[tblField.unsafeName] = columnNameWithTypeSuffix;
+        if (!columnNamesFromWarehouse.includes(columnNameWithTypeSuffix)) {
+          columnsToBeAddedInWarehouse.push({
+            unsafeName: columnNameWithTypeSuffix,
+            name: columnNameWithTypeSuffix,
+            definition: tblField.definition
+          });
+        }
+      });
+      if (columnsToBeAddedInWarehouse.length > 0) {
+        await this.addColumns(columnsToBeAddedInWarehouse);
+      }
+    }
+  };
+  updateTableColumns = async () => {
+    const columnsFromWarehouse = await this.getTableColumnsFromWarehouse();
+    const columnNamesFromWarehouse = columnsFromWarehouse.map((col) => col.name);
+    const columnsMappingStore = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    const columnsToAdd = Object.keys(columnsMappingStore).filter((unsafeColumnKey) => {
+      const safeColumnName = columnsMappingStore[unsafeColumnKey];
+      if (!safeColumnName) {
+        return false;
+      }
+      return !columnNamesFromWarehouse.includes(safeColumnName);
+    }).map((unsafeColumnKey) => {
+      const safeColumnName = columnsMappingStore[unsafeColumnKey];
+      const definition = this.flattenedSchema[unsafeColumnKey];
+      if (!safeColumnName || !definition) {
+        return void 0;
+      }
+      return {
+        unsafeName: unsafeColumnKey,
+        name: safeColumnName,
+        definition
+      };
+    }).filter((item) => !!item);
+    await this.addColumns(columnsToAdd);
+    const columnsToVersion = Object.keys(this.flattenedSchema).filter((unsafeColumnKey) => {
+      const definition = this.flattenedSchema[unsafeColumnKey];
+      if (!definition) {
+        return false;
+      }
+      const warehouseTypeToBeUsed = this.getWarehouseTypeFromJSONSchema(definition);
+      if (warehouseTypeToBeUsed) {
+        const safeName = this.safeColumnName(unsafeColumnKey);
+        if (columnNamesFromWarehouse.includes(safeName)) {
+          const columnFromWarehouse = columnsFromWarehouse.find((col) => col.name === safeName);
+          if (!columnFromWarehouse) {
+            throw new Error(`We can't find in Warehouse schema any column definition for colName: ${safeName}`);
+          }
+          const { type } = columnFromWarehouse;
+          if (warehouseTypeToBeUsed !== type) {
+            return true;
+          }
+        }
+      }
+      return false;
+    }).map((unsafeColumnKey) => {
+      const definition = this.flattenedSchema[unsafeColumnKey];
+      if (!definition) {
+        return void 0;
+      }
+      return {
+        unsafeName: unsafeColumnKey,
+        name: this.safeColumnName(unsafeColumnKey),
+        definition
+      };
+    }).filter((tableField) => !!tableField);
+    await this.addVersionedColumns(columnsToVersion);
+  };
+  mergeTempTableWithDestination = async (replicationMethod) => {
+    let mergeQueries = [];
+    if (replicationMethod === "FULL_TABLE") {
+      mergeQueries = this.getReplaceQueries();
+    } else if (replicationMethod === "INCREMENTAL") {
+      mergeQueries = this.getMergeQueries();
+    } else if (replicationMethod === "LOG_BASED") {
+      mergeQueries = this.getMergeQueries();
+    }
+    logger.info(`\u{1F346} Table=${this.table} - Merging staging data with method ${replicationMethod}`);
+    logger.debug(`Table=${this.table} - Merge queries: %j`, mergeQueries);
+    await this.runQueriesWithRetry(mergeQueries);
+  };
+};
+
+// src/sdk/models/target/error.ts
+var SchemaValidationError = class extends Error {
+  streamId;
+  validationErrors;
+  constructor(message, streamId, errors2) {
+    super(message);
+    this.name = "SchemaValidationError";
+    this.streamId = streamId;
+    this.validationErrors = errors2;
+  }
+};
+var MissingFieldInSchemaError = class extends Error {
+  streamId;
+  constructor(message, streamId) {
+    super(message);
+    this.name = "MissingFieldInSchemaError";
+    this.streamId = streamId;
+  }
+};
+var MissingSchemaError = class extends Error {
+  streamId;
+  constructor(message, streamId) {
+    super(message);
+    this.name = "MissingSchemaError";
+    this.streamId = streamId;
+  }
+};
+
+// src/sdk/models/target/record.ts
+var removeParasiteProperties = (record, schema) => {
+  const schemaKeys = Object.keys(schema);
+  Object.keys(record).forEach((key) => {
+    if (!schemaKeys.includes(key)) {
+      delete record[key];
+    }
+  });
+  return record;
+};
+var addWhalyFields = (record, batchDate) => {
+  return {
+    ...record,
+    _whaly_synced: batchDate
+  };
+};
+
+// src/sdk/models/target/renameColumnStore.ts
+var RenameColumnStore = class {
+  // Store all the mappings store for all streamId
+  renamedColumns;
+  safeColumnNameConverter;
+  constructor() {
+    this.renamedColumns = {};
+  }
+  setSafeColumnNameConverter = (fn) => {
+    this.safeColumnNameConverter = fn;
+  };
+  // This function has a side effect on:
+  //  - `this.renamedColumns`
+  // This isn't very clean and should be updated to write a pure version of this function
+  computeColumnNameForStream = (streamId, flattenedSchema) => {
+    const safeColNameConv = this.safeColumnNameConverter;
+    if (!safeColNameConv) {
+      throw new Error(`safeColumnNameConverter is not defined!`);
+    }
+    const colSafeToUnsafeMapping = Object.keys(flattenedSchema).sort().reduce((acc, unsafeColName) => {
+      const safeColName = safeColNameConv(unsafeColName);
+      if (!acc[safeColName]) {
+        acc[safeColName] = unsafeColName;
+      } else {
+        logger.info(`\u{1F6A8} Stream: ${streamId} - We have a column name conflict after sanitization for column unsafe=${unsafeColName}. We'll rename the sanitized name to a non taken name.
+                Details: unsafeColName=${unsafeColName} is colliding with: ${acc[safeColName]}`);
+        for (let i = 1; ; i++) {
+          if (!acc[`${safeColName}_${i}`]) {
+            acc[`${safeColName}_${i}`] = unsafeColName;
+            break;
+          }
+        }
+      }
+      return acc;
+    }, {});
+    const colUnsafeToSafeMapping = Object.keys(colSafeToUnsafeMapping).reduce((acc, safeColName) => {
+      const unsafeColname = colSafeToUnsafeMapping[safeColName];
+      if (unsafeColname) {
+        acc[unsafeColname] = safeColName;
+      }
+      return acc;
+    }, {});
+    const mapping = this.renamedColumns[streamId] || {};
+    this.renamedColumns[streamId] = mapping;
+    Object.keys(colUnsafeToSafeMapping).forEach((unsafeColName) => {
+      const safeName = colUnsafeToSafeMapping[unsafeColName];
+      if (safeName) {
+        mapping[unsafeColName] = safeName;
+      }
+    });
+  };
+  getColumnTranslation = (streamId, originalColumnName) => {
+    if (!this.renamedColumns[streamId]) {
+      throw new Error(
+        `StreamId=${streamId} - Columns Mapping Store is not initialized! 
+                Trying to get renamed value for column '${originalColumnName}'`
+      );
+    }
+    const value = this.renamedColumns[streamId][originalColumnName];
+    if (!value) {
+      throw new Error(`StreamId=${streamId} - No renamed value found for column '${originalColumnName}'`);
+    }
+    return value;
+  };
+  getUnsafeToSafeColumnMapping = (streamId) => {
+    const mapping = this.renamedColumns[streamId];
+    if (!mapping) {
+      throw new Error(`StreamId=${streamId} - Columns Mapping Store is not initialized!`);
+    }
+    return mapping;
+  };
+  isReady = (streamId) => {
+    if (!this.renamedColumns[streamId]) {
+      return false;
+    } else {
+      return true;
+    }
+  };
+};
+
+// src/sdk/models/target/schema.ts
+var flattenSchema = (streamId, schema) => {
+  logger.debug(`Flattening schema for stream: ${streamId}`);
+  try {
+    const mSchema = schema;
+    if (!mSchema["properties"]) {
+      return {};
+    }
+    const properties = mSchema["properties"];
+    const flattenedSchema = Object.keys(properties).reduce((acc, propertyName) => {
+      const propertySchema = properties[propertyName];
+      const columnName = propertyName;
+      if (!propertySchema) {
+        return acc;
+      }
+      if (!propertySchema["type"]) {
+        throw new Error(`We don't have any \`type\` for the property: \`${propertyName}\`.
+            Schema of the property is: ${JSON.stringify(propertySchema)}`);
+      }
+      acc[columnName] = {
+        type: propertySchema["type"],
+        format: propertySchema["format"],
+        items: propertySchema["items"]
+      };
+      return acc;
+    }, {});
+    return flattenedSchema;
+  } catch (err) {
+    logger.error(`There was an error when processing the schema of stream:\`${streamId}\`: err: ${err.message} - ${err.stack}.
+    Processed schema: ${JSON.stringify(schema)}`);
+    gracefulExit(1);
+    return {};
+  }
+};
+
+// src/sdk/models/target/target.ts
+var import_jsonschema = require("jsonschema");
+var import_moment4 = __toESM(require("moment"));
+
+// src/sdk/models/target/streamDbState.ts
+var import_moment3 = __toESM(require("moment"));
+
+// src/sdk/models/target/temporaryFile.ts
+var import_fs_extra2 = __toESM(require("fs-extra"));
+var import_lodash = require("lodash");
+var createTemporaryFileStream = (streamId) => {
+  import_fs_extra2.default.ensureDirSync("./tmp");
+  const path = `./tmp/${safePath(streamId)}-${(0, import_lodash.uniqueId)()}.tmp`;
+  const writeStream = import_fs_extra2.default.createWriteStream(path, { flags: "w" });
+  return {
+    stream: writeStream,
+    path
+  };
+};
+function safePath(path) {
+  let formattedPath = path;
+  formattedPath = formattedPath.replace(/^The\s+/gi, "");
+  formattedPath = formattedPath.replace(/\s+(?:\(|\[)?Dis(?:c|k) (?:One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\d)+(?:\)|\])?/gi, "");
+  formattedPath = formattedPath.replace(/\s+/gi, "_");
+  formattedPath = formattedPath.replace(/(?:\'|`)/gi, "");
+  formattedPath = replaceAccented(formattedPath);
+  formattedPath = replaceSymbols(formattedPath);
+  formattedPath = formattedPath.replace(/(?:[^a-z0-9])+/gi, "_");
+  return formattedPath;
+}
+function replaceAccented(str) {
+  str = str.replace(/(?:\xC0|\xC1|\xC2|\xC3|\xC4|\xC5)/g, "A");
+  str = str.replace(/(?:\xC7)/g, "C");
+  str = str.replace(/(?:\xC8|\xC9|\xCA|\xCB)/g, "E");
+  str = str.replace(/(?:\xCC|\xCD|\xCE|\xCF)/g, "I");
+  str = str.replace(/(?:\xD2|\xD3|\xD4|\xD6|\xD5)/g, "O");
+  str = str.replace(/(?:\xD9|\xDA|\xDB|\xDC)/g, "U");
+  str = str.replace(/(?:\xDD|\x9F)/g, "Y");
+  str = str.replace(/(?:\xD1)/g, "N");
+  str = str.replace(/(?:\xC6)/g, "Ae");
+  str = str.replace(/(?:\xDF)/g, "ss");
+  str = str.replace(/(?:\xE0|\xE1|\xE2|\xE4|\xE5|\xE3)/g, "a");
+  str = str.replace(/(?:\xE7)/g, "c");
+  str = str.replace(/(?:\xE8|\xE9|\xEA|\xEB)/g, "e");
+  str = str.replace(/(?:\xEC|\xED|\xEE|\xEF)/g, "i");
+  str = str.replace(/(?:\xF2|\xF3|\xF4|\xF6|\xF5)/g, "o");
+  str = str.replace(/(?:\xF9|\xFA|\xFB|\xFC)/g, "u");
+  str = str.replace(/(?:\xFD|\xFF)/g, "y");
+  str = str.replace(/(?:\xF1)/g, "n");
+  str = str.replace(/(?:\xE6)/g, "ae");
+  return str;
+}
+function replaceSymbols(str) {
+  str = str.replace(/(?:\(|\))/g, "");
+  str = str.replace(/(?:\.|\?)/g, "");
+  str = str.replace(/(?:_+)/g, "_");
+  str = str.replace(/(?:\$+)/g, "s");
+  return str;
+}
+
+// src/sdk/models/target/streamDbState.ts
+var fs4 = __toESM(require("fs"));
+var StreamState = class {
+  streamId;
+  schema;
+  dbSync;
+  batchDate;
+  // Count rows batched in the local file, waiting for Warehouse upload
+  batchedRowCount;
+  // Local CSV files written to be sent to Warehouse
+  fileToLoad;
+  // Count rows sent to Warehouse
+  syncedRowCount;
+  keyProperties;
+  // When a FULL_TABLE stream is loaded multiple times, the query used for the 2nd+ loads
+  // needs to be different than the 1st merge query -> 1st load is dropping the existing table, 2nd+ loads are merging with the existing table (otherwise the previous load data are dropped)
+  // So we need to keep track of whether we are at the first load or not
+  _hasBeenLoadedYetDuringThisSync;
+  replicationMethod;
+  constructor(streamId, dbSync, replicationMethod) {
+    this.streamId = streamId;
+    this.dbSync = dbSync;
+    this.batchedRowCount = 0;
+    this.fileToLoad = createTemporaryFileStream(streamId);
+    this.syncedRowCount = 0;
+    this.keyProperties = [];
+    this.batchDate = (0, import_moment3.default)();
+    this.replicationMethod = replicationMethod;
+    this._hasBeenLoadedYetDuringThisSync = false;
+  }
+  setSchema(schema) {
+    this.schema = schema;
+  }
+  getSchema() {
+    return this.schema;
+  }
+  getBatchDate() {
+    return this.batchDate.format(defaultDateTimeFormat);
+  }
+  getBatchedRowCount() {
+    return this.batchedRowCount;
+  }
+  incrementBatchedRowCount() {
+    this.batchedRowCount += 1;
+  }
+  getFileToLoad() {
+    return this.fileToLoad;
+  }
+  resetFileToLoad() {
+    this.fileToLoad.stream.close();
+    const oldPath = this.fileToLoad.path;
+    fs4.unlinkSync(oldPath);
+    this.fileToLoad = createTemporaryFileStream(this.streamId);
+    this.batchedRowCount = 0;
+  }
+  getKeyProperties() {
+    return this.keyProperties;
+  }
+  setKeyProperties(keyProperties) {
+    this.keyProperties = keyProperties;
+  }
+  getDbSync() {
+    return this.dbSync;
+  }
+  getReplicationMethod() {
+    return this.replicationMethod;
+  }
+  setReplicationMethod(r) {
+    this.replicationMethod = r;
+  }
+  setHasBeenLoadedYet() {
+    this._hasBeenLoadedYetDuringThisSync = true;
+  }
+};
+
+// src/sdk/models/target/target.ts
+var import_async_mutex = require("async-mutex");
+var import_bluebird3 = require("bluebird");
+var semaphore = new import_async_mutex.Semaphore(1);
+var ITarget = class _ITarget {
+  config;
+  schemaHooks;
+  resolver;
+  syncTime;
+  // Latest state message received from the Tap
+  // Not yet flushed as we didn't upload the stream data since receiving it 
+  batchedState;
+  // Latest "flushed" state message
+  // A state message is flushed when we do a stream upload.
+  flushedState;
+  renameColumnStore;
+  streams;
+  // JSON Schema validator
+  validator;
+  constructor(config, resolver) {
+    this.config = config;
+    this.resolver = resolver;
+    this.schemaHooks = resolver.getSchemaHooks();
+    this.streams = {};
+    this.batchedState = {};
+    this.flushedState = {};
+    this.validator = new import_jsonschema.Validator();
+    this.syncTime = (0, import_moment4.default)();
+    this.renameColumnStore = new RenameColumnStore();
+  }
+  ///////////////////////////////////////////
+  //// To be overriden by child classes /////
+  ///////////////////////////////////////////
+  static requiredConfigKeys = [];
+  ////////////////////////////////////////////
+  /////////// Implementation /////////////////
+  ////////////////////////////////////////////
+  // Analyze schema changes to determine if they are breaking or non-breaking
+  analyzeSchemaChanges = (previousSchema, newSchema, streamId) => {
+    const changes = [];
+    let hasBreakingChanges = false;
+    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => !col.startsWith("_whaly")));
+    const newColumns = new Set(Object.keys(newSchema).filter((col) => !col.startsWith("_whaly")));
+    for (const columnName of newColumns) {
+      if (!prevColumns.has(columnName)) {
+        changes.push(`Added column '${columnName}'`);
+      }
+    }
+    for (const columnName of prevColumns) {
+      if (!newColumns.has(columnName)) {
+        changes.push(`Removed column '${columnName}'`);
+        hasBreakingChanges = true;
+      }
+    }
+    for (const columnName of prevColumns) {
+      if (newColumns.has(columnName)) {
+        const prevType = previousSchema[columnName];
+        const newType = newSchema[columnName];
+        if (JSON.stringify(prevType) !== JSON.stringify(newType)) {
+          changes.push(`Changed type of column '${columnName}'`);
+          hasBreakingChanges = true;
+        }
+      }
+    }
+    if (changes.length === 0) {
+      changes.push("No schema changes detected");
+    }
+    logger.debug(`StreamId=${streamId} - Schema analysis: ${hasBreakingChanges ? "BREAKING" : "NON-BREAKING"} changes found: ${changes.join(", ")}`);
+    return { hasBreakingChanges, changes };
+  };
+  // Serialize a RECORD message into a line that will be uploaded in the staging area
+  getSerializedRecord = (streamId, record) => {
+    const stream = this.streams[streamId];
+    if (!stream) {
+      throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
+    }
+    return stream.getDbSync().getSerializedRecord(record);
+  };
+  record = async (message) => {
+    try {
+      const streamId = message.stream;
+      const stream = this.streams[streamId];
+      if (!stream) {
+        throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
+      }
+      await semaphore.runExclusive(async () => {
+        const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
+        if (shouldUploadIntermediateBatch) {
+          await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
+        }
+      });
+      const record = message.record;
+      const schema = stream.getSchema();
+      if (!schema) {
+        logger.warn(`Stream: ${streamId} - Received a RECORD message while schema is undefined. Dropping message.`);
+        return;
+      }
+      const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
+      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
+      const validationResult = this.validator.validate(recordWithWhalyFields, schema);
+      if (!validationResult.valid) {
+        throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
+                Validation errors: ${JSON.stringify(validationResult.errors)}
+
+                Record: ${JSON.stringify(recordWithoutParasiteProperties)}
+                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validationResult.errors);
+      }
+      const recordWithValidDate = this.validateDateRange(recordWithWhalyFields, schema);
+      const recordWithDecimal = this.convertNumberIntoDecimal(recordWithValidDate, schema);
+      const dbSyncInstance = stream.getDbSync();
+      const recordWithrenamedColumns = dbSyncInstance.renameColumns(recordWithDecimal);
+      const serializedRecord = this.getSerializedRecord(streamId, recordWithrenamedColumns);
+      return new Promise((resolve, reject) => {
+        stream.getFileToLoad().stream.write(serializedRecord, (error) => {
+          if (error) {
+            reject(error);
+          }
+          stream.incrementBatchedRowCount();
+          resolve();
+        });
+      });
+    } catch (err) {
+      logger.error(`Error when handling "record" message`);
+      throw err;
+    }
+  };
+  initStreamStateAndSyncService(streamId, replicationMethod) {
+    const dbSyncInstance = this.newDBSyncInstance(
+      streamId,
+      this.config.database,
+      this.config.schema,
+      this.genTableName(streamId)
+    );
+    this.streams[streamId] = new StreamState(
+      streamId,
+      dbSyncInstance,
+      replicationMethod || "FULL_TABLE"
+    );
+  }
+  replicationMethod = (message) => {
+    try {
+      logger.info(`\u{1F448} Received replicationMethod message: %j`, message);
+      const streamId = message.stream;
+      if (!this.streams[streamId]) {
+        this.initStreamStateAndSyncService(streamId, message.replication);
+      } else {
+        this.streams[streamId].setReplicationMethod(message.replication);
+      }
+    } catch (err) {
+      logger.error(`Error when handling "replicationMethod" event`);
+      throw err;
+    }
+  };
+  schema = async (message) => {
+    try {
+      const streamId = message.stream;
+      logger.info(`Stream: ${streamId} - Got a SCHEMA message from the tap`);
+      logger.debug(`Stream: ${streamId} - Got a SCHEMA message from the tap: %j`, message);
+      if (!this.streams[streamId]) {
+        this.initStreamStateAndSyncService(streamId);
+      } else {
+        const prevStreamState = this.streams[streamId];
+        if (prevStreamState.batchedRowCount > 0) {
+          const prevSchema = prevStreamState.getSchema();
+          if (prevSchema) {
+            const newFlattenedSchema = flattenSchema(streamId, message.schema);
+            const { hasBreakingChanges, changes } = this.analyzeSchemaChanges(prevSchema, newFlattenedSchema, streamId);
+            if (hasBreakingChanges) {
+              logger.info(`\u{1F525} StreamId=${streamId} - Detected BREAKING schema changes: ${changes.join(", ")}. Loading batched records before applying new schema.`);
+              await _ITarget.uploadSingleStreamToWarehouse(prevStreamState);
+            } else {
+              logger.info(`\u2705 StreamId=${streamId} - Detected NON-BREAKING schema changes: ${changes.join(", ")}. Continuing without uploading batched records.`);
+            }
+          }
+        }
+      }
+      const streamState = this.streams[streamId];
+      if (!streamState) {
+        throw new MissingSchemaError(`Stream: ${streamId} - Received SCHEMA message before stream state was initialized.`, streamId);
+      }
+      const schema = flattenSchema(streamId, message.schema);
+      schema["_whaly_synced"] = {
+        format: "date-time",
+        type: ["null", "string"]
+      };
+      const dbSync = streamState.getDbSync();
+      Object.keys(schema).forEach((fieldUnsafeKey) => {
+        const fieldDef = schema[fieldUnsafeKey];
+        const targetWarehouseType = fieldDef ? dbSync.getWarehouseTypeFromJSONSchema(fieldDef) : void 0;
+        if (!targetWarehouseType) {
+          delete schema[fieldUnsafeKey];
+        }
+      });
+      streamState.setSchema(schema);
+      if (message.keyProperties.length === 0) {
+        throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
+            Received SCHEMA message: ${JSON.stringify(message)}`);
+      }
+      streamState.setKeyProperties(message.keyProperties);
+      await dbSync.updateSchemaInWarehouse(schema, message.keyProperties);
+      const {
+        database: databaseName,
+        schema: schemaName
+      } = this.config;
+      const tableName = this.genTableName(streamId);
+      const formattedRelationshipMessage = {
+        ...message,
+        keyProperties: message.keyProperties.map((k) => {
+          const translation = streamState.getDbSync().renamedColumnStore.getColumnTranslation(message.stream, k);
+          if (translation) {
+            return translation;
+          }
+          return k;
+        }),
+        relationships: {
+          left: message.relationships.left.flatMap((l) => {
+            if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
+              const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
+              const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
+              if (from && to) {
+                return [{
+                  type: l.type,
+                  streamId: l.streamId,
+                  from,
+                  to
+                }];
+              }
+            }
+            return [];
+          }),
+          right: message.relationships.right.flatMap((r) => {
+            if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
+              const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
+              const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
+              if (from && to) {
+                return [{
+                  type: r.type,
+                  streamId: r.streamId,
+                  from,
+                  to
+                }];
+              }
+            }
+            return [];
+          })
+        }
+      };
+      await import_bluebird3.Promise.map(this.schemaHooks, async (hook) => {
+        const input = {
+          databaseName,
+          schemaName,
+          tableName,
+          message: formattedRelationshipMessage
+        };
+        await hook.writeSchema(input);
+      }, { concurrency: 3 });
+      return Promise.resolve();
+    } catch (err) {
+      logger.error(`Error when handling schema event.`, err);
+      throw err;
+    }
+  };
+  state = (message) => {
+    try {
+      this.batchedState = message.value;
+      if (!this.flushedState) {
+        this.flushedState = Object.assign({}, this.state);
+      }
+      return Promise.resolve();
+    } catch (err) {
+      logger.error(`Error while handling state event.`);
+      throw err;
+    }
+  };
+  complete = async () => {
+    try {
+      logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
+      await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
+      const configResolver = loadResolver();
+      await configResolver.markSyncComplete();
+      await configResolver.flushMetrics();
+      return Promise.resolve();
+    } catch (err) {
+      logger.error(`Error while handling complete event.`);
+      throw err;
+    }
+  };
+  shouldUploadIntermediateBatch = (streamId) => {
+    const stream = this.streams[streamId];
+    if (!stream) {
+      return false;
+    }
+    const batchedRecords = stream.getBatchedRowCount();
+    if (stream.getReplicationMethod() === "INCREMENTAL" && batchedRecords > 1e6) {
+      logger.info(`\u{1F9CA} Stream=${streamId} has reached the maximum amount of batched records. We'll trigger a flush of ALL streams in the Warehouse.`);
+      return true;
+    } else {
+      return false;
+    }
+  };
+  loadAllStreamsInWarehouse = async (opts) => {
+    logger.info(`\u{1F69B}\u{1F69B}\u{1F69B} Loading ALL Streams in Warehouse.`);
+    const { isFinalLoad } = opts;
+    const streamsWithRemainingRecords = Object.keys(this.streams).filter((streamId) => {
+      const s = this.streams[streamId];
+      if (!s) {
+        return false;
+      }
+      if (!isFinalLoad && s.getReplicationMethod() === "FULL_TABLE") {
+        return false;
+      }
+      return s.getBatchedRowCount() > 0;
+    });
+    if (streamsWithRemainingRecords.length > 0) {
+      this.flushedState = await this.uploadStreamsToWarehouse(streamsWithRemainingRecords);
+    }
+    if (Object.keys(this.flushedState).length > 0) {
+      await this.emitState(Object.assign({}, this.flushedState));
+    } else {
+      await this.emitState(Object.assign({}, this.state));
+    }
+  };
+  emitState = (state) => {
+    const configResolver = loadResolver();
+    return writeStateFile(configResolver, JSON.stringify(state));
+  };
+  uploadStreamsToWarehouse = async (streamsToUpload) => {
+    try {
+      await import_bluebird3.Promise.map(streamsToUpload, async (streamId) => {
+        logger.info(`\u{1F4E4} Upload stream: ${streamId} to Warehouse`);
+        const streamState = this.streams[streamId];
+        if (streamState) {
+          await _ITarget.uploadSingleStreamToWarehouse(streamState);
+        }
+      }, { concurrency: 3 });
+      return this.batchedState;
+    } catch (err) {
+      logger.error(`Error while uploading all streams + merging them.`);
+      throw err;
+    }
+  };
+  static uploadSingleStreamToWarehouse = async (streamState) => {
+    const dbSync = streamState.getDbSync();
+    try {
+      const replicationMethod = streamState.getReplicationMethod();
+      const fileToLoad = streamState.getFileToLoad();
+      await dbSync.createStagingArea();
+      await dbSync.loadStreamInStagingArea(fileToLoad.path);
+      await dbSync.mergeTempTableWithDestination(replicationMethod);
+      await dbSync.deleteStagingArea();
+      streamState.setHasBeenLoadedYet();
+      streamState.resetFileToLoad();
+    } catch (err) {
+      if (err instanceof Error) {
+        logger.error(`StreamId=${dbSync.streamId} - Couldn't upload stream due to an error: %s`, err.message);
+        throw err;
+      } else {
+        throw new Error("Unknown error");
+      }
+    }
+  };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ALL_STREAM_ID_KEYWORD,
+  API_CALLS_METRIC_NAME,
+  Authenticator,
+  BATCH_INTERVAL_MS,
+  Catalog,
+  CounterMetric,
+  DEFAULT_MAX_CONCURRENT_STREAMS,
+  EXECUTION_TIME_METRIC_NAME,
+  ITarget,
+  MissingFieldInSchemaError,
+  MissingSchemaError,
+  RESTStreamV2,
+  ROWS_SYNCED_METRIC_NAME,
+  RecordMessage,
+  RenameColumnStore,
+  ReplicationMethodMessage,
+  Resolver,
+  SchemaMessage,
+  SchemaValidationError,
+  StateMessage,
+  StateService,
+  StreamMetadata,
+  StreamV2,
+  StreamWarehouseSyncService,
+  Tap,
+  _greaterThan,
+  addWhalyFields,
+  checkRequiredConfigKeys,
+  createTemporaryFileStream,
+  extractStateForStream,
+  finalizeStateProgressMarkers,
+  findRelatedRelationships,
+  flattenSchema,
+  getAllMetrics,
+  getAxiosInstance,
+  getCounterMetric,
+  getCounterMetrics,
+  getDownloadFileApiCall,
+  getJSONApiCall,
+  getJSONApiCallWithFullResponse,
+  getRawJSONApiCall,
+  gracefulExit,
+  haltAndCatchFire,
+  incrementStreamState,
+  loadJson,
+  loadSchema,
+  loadShoreConfig,
+  logger,
+  postFormDataApiCall,
+  postJSONApiCall,
+  postUrlEncodedApiCall,
+  printMemoryFootprint,
+  removeParasiteProperties,
+  safePath,
+  writeCatalogFile,
+  writeStateFile
+});
+//# sourceMappingURL=index.js.map