@whaly/connector-sdk 0.1.0 → 0.1.1

package/dist/index.js

@@ -34,14 +34,14 @@ __export(index_exports, {
   API_CALLS_METRIC_NAME: () => API_CALLS_METRIC_NAME,
   Authenticator: () => Authenticator,
   BATCH_INTERVAL_MS: () => BATCH_INTERVAL_MS,
-  Catalog: () => Catalog,
+  BigQueryTarget: () => BigQueryTarget,
   CounterMetric: () => CounterMetric,
   DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
   ITarget: () => ITarget,
   MissingFieldInSchemaError: () => MissingFieldInSchemaError,
   MissingSchemaError: () => MissingSchemaError,
-  RESTStreamV2: () => RESTStreamV2,
+  RESTStream: () => RESTStream,
   ROWS_SYNCED_METRIC_NAME: () => ROWS_SYNCED_METRIC_NAME,
   RecordMessage: () => RecordMessage,
   RenameColumnStore: () => RenameColumnStore,
@@ -51,13 +51,12 @@ __export(index_exports, {
   SchemaValidationError: () => SchemaValidationError,
   StateMessage: () => StateMessage,
   StateService: () => StateService,
+  Stream: () => Stream,
   StreamMetadata: () => StreamMetadata,
-  StreamV2: () => StreamV2,
   StreamWarehouseSyncService: () => StreamWarehouseSyncService,
   Tap: () => Tap,
   _greaterThan: () => _greaterThan,
   addWhalyFields: () => addWhalyFields,
-  checkRequiredConfigKeys: () => checkRequiredConfigKeys,
   createTemporaryFileStream: () => createTemporaryFileStream,
   extractStateForStream: () => extractStateForStream,
   finalizeStateProgressMarkers: () => finalizeStateProgressMarkers,
@@ -75,8 +74,8 @@ __export(index_exports, {
   haltAndCatchFire: () => haltAndCatchFire,
   incrementStreamState: () => incrementStreamState,
   loadJson: () => loadJson,
+  loadResolver: () => loadResolver,
   loadSchema: () => loadSchema,
-  loadShoreConfig: () => loadShoreConfig,
   logger: () => logger,
   postFormDataApiCall: () => postFormDataApiCall,
   postJSONApiCall: () => postJSONApiCall,
@@ -84,7 +83,6 @@ __export(index_exports, {
   printMemoryFootprint: () => printMemoryFootprint,
   removeParasiteProperties: () => removeParasiteProperties,
   safePath: () => safePath,
-  writeCatalogFile: () => writeCatalogFile,
   writeStateFile: () => writeStateFile
 });
 module.exports = __toCommonJS(index_exports);
@@ -116,10 +114,10 @@ var readAndParseJSONFile = (path) => {
 };
 
 // src/resolvers/local-file/hook/schemaHook.ts
-var import_fs_extra = require("fs-extra");
+var import_fs_extra = __toESM(require("fs-extra"));
 var LocalSchemaHook = class {
   async writeSchema(input) {
-    await (0, import_fs_extra.appendFile)("./schema.tmp", JSON.stringify(input) + "\n");
+    await import_fs_extra.default.appendFile("./schema.tmp", JSON.stringify(input) + "\n");
   }
 };
 
@@ -180,15 +178,9 @@ var CounterMetric = class {
 
 // src/resolvers/local-file/resolver.ts
 var { combine, prettyPrint, timestamp, errors, splat } = import_winston.default.format;
-var configPath = "./config.json";
-var targetConfigPath = "./target-config.json";
-var catalogPath = "./catalog.json";
 var statePath = "./state.json";
 var metricsPath = "./metrics.json";
 var LocalFilesResolver = class extends Resolver {
-  startVPNIfNeeded() {
-    return Promise.resolve();
-  }
   checkIfCanSync() {
     return Promise.resolve(true);
   }
@@ -213,49 +205,15 @@ var LocalFilesResolver = class extends Resolver {
   constructor() {
     super();
   }
-  getConfig(command) {
+  getState() {
     logger.info(`\u{1F4C1} Using the LOCAL resolver`);
-    const config = readAndParseJSONFile(configPath);
-    const targetConfig = readAndParseJSONFile(targetConfigPath);
-    const destination = process.env.SHORE__DESTINATION;
-    if (!destination) {
-      throw new Error(`\u274C Env variable \`SHORE__DESTINATION\` is not set.
-            Did you forget to configure it?
-            
-            Possible values are: \`GOOGLE_BIGQUERY\`, \`SNOWFLAKE\``);
-    }
-    if (command === `READ`) {
-      if (destination === `GOOGLE_BIGQUERY`) {
-        const googleCredentials = process.env.GOOGLE_APPLICATION_CREDENTIALS;
-        if (!googleCredentials) {
-          throw new Error(`Env variable \`GOOGLE_APPLICATION_CREDENTIALS\` is not set. 
-                Did you forget to configure it?
-
-                This variable should link to a file containing the proper Google Service Account credentials.
-                `);
-        }
-      }
-      const catalog = readAndParseJSONFile(catalogPath);
-      const state = readAndParseJSONFile(statePath);
-      return Promise.resolve({
-        command,
-        destination,
-        config,
-        targetConfig,
-        catalog,
-        state
-      });
-    } else {
-      return Promise.resolve({
-        command,
-        destination,
-        config,
-        targetConfig
-      });
-    }
+    const state = readAndParseJSONFile(statePath);
+    return Promise.resolve({
+      state
+    });
   }
   writeCatalog(catalog) {
-    return writeFile(catalogPath, JSON.stringify(catalog));
+    return Promise.resolve();
   }
   writeState(state) {
     return writeFile(statePath, state);
@@ -286,9 +244,7 @@ var LocalFilesResolver = class extends Resolver {
   }
   updateSourceValue(optionKey, optionValue) {
     logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
-    const config = readAndParseJSONFile(configPath);
-    config[optionKey] = optionValue;
-    return writeFile(configPath, JSON.stringify(config));
+    return Promise.resolve();
   }
   getLogFormat() {
     return combine(
@@ -301,29 +257,8 @@ var LocalFilesResolver = class extends Resolver {
 };
 
 // src/sdk/service/resolver.ts
-function initResolver(resolverType) {
-  if (resolverType === "LOCAL") {
-    return new LocalFilesResolver();
-  } else {
-    throw new Error(`\u274C Unsupported configResolver: ${resolverType}`);
-  }
-}
 var loadResolver = () => {
-  const configResolver = process.env.SHORE__CONFIG_RESOLVER;
-  if (!configResolver) {
-    throw new Error(`Env variable \`SHORE__CONFIG_RESOLVER\` is not set. 
-        Did you forget to configure it?
-
-        Possible values are \`LOCAL\`, \`WHALY\`.
-        Please read the documentation to get the full description of the behavior of each.
-        `);
-  }
-  if (configResolver === "WHALY" || configResolver === "LOCAL") {
-    return initResolver(configResolver);
-  } else {
-    throw new Error(`Resolver: ${configResolver} is not supported.
-        Did you properly configure \`SHORE__CONFIG_RESOLVER\`?`);
-  }
+  return new LocalFilesResolver();
 };
 
 // src/sdk/service/logger.ts
@@ -373,30 +308,6 @@ var writeStateFile = (resolver, state) => {
   logger.info(`\u{1F4DD} Writing the state file.`);
   return resolver.writeState(state);
 };
-var writeCatalogFile = (resolver, catalog) => {
-  logger.info(`\u{1F4DD} Writing the catalog`);
-  return resolver.writeCatalog(catalog);
-};
-var loadShoreConfig = (resolver) => {
-  const command = process.env.SHORE__COMMAND;
-  if (!command) {
-    throw new Error(`Env variable \`SHORE__COMMAND\` is not set. 
-        Did you forget to configure it?
-
-        Possible values are: \`DISCOVER\`, \`READ\``);
-  }
-  if (command === `DISCOVER`) {
-    logger.info(`\u{1F50E} Mode: discovery
-        Shore will detect the available streams in the source and their configuration.`);
-  } else if (command === `READ`) {
-    logger.info(`\u{1F501} Mode: Synchronizing data
-        Shore will extract and push data from the source to the destination.`);
-  } else {
-    throw new Error(`Command: \`${command}\` is not supported.
-        Did you properly configured env variable \`SHORE__COMMAND\`?`);
-  }
-  return resolver.getConfig(command);
-};
 function readFile(fileName, filePath) {
   try {
     return (0, import_fs2.readFileSync)(filePath);
@@ -422,13 +333,6 @@ var loadSchema = (streamId) => {
   const schema = loadJson(`schemas/${streamId}.json`);
   return schema;
 };
-var checkRequiredConfigKeys = (args, requiredConfigKeys) => {
-  requiredConfigKeys.forEach((configKey) => {
-    if (!args[configKey]) {
-      throw new Error(`Config is missing required key: ${configKey}`);
-    }
-  });
-};
 
 // src/sdk/service/network.ts
 var import_axios = __toESM(require("axios"));
@@ -706,6 +610,102 @@ var findRelatedRelationships = (streamId, allRels) => {
   }, { left: [], right: [] });
 };
 
+// src/sdk/models/messages.ts
+var RecordMessage = class {
+  type = "RECORD";
+  stream;
+  record;
+  constructor(record, stream) {
+    this.record = record;
+    this.stream = stream;
+  }
+  toString() {
+    const result = {
+      type: "RECORD",
+      record: this.record,
+      stream: this.stream
+    };
+    return JSON.stringify(result);
+  }
+};
+var StateMessage = class {
+  type = "STATE";
+  value;
+  constructor(value) {
+    this.value = value;
+  }
+  toString() {
+    const result = {
+      type: "STATE",
+      value: this.value
+    };
+    return JSON.stringify(result);
+  }
+};
+var ReplicationMethodMessage = class {
+  type = "REPLICATION_METHOD";
+  stream;
+  replication;
+  constructor(stream, version) {
+    this.stream = stream;
+    this.replication = version;
+  }
+  toString() {
+    const result = {
+      type: "REPLICATION_METHOD",
+      replication: this.replication,
+      stream: this.stream
+    };
+    return JSON.stringify(result);
+  }
+};
+var SchemaMessage = class {
+  type = "SCHEMA";
+  stream;
+  schema;
+  keyProperties;
+  bookmarkProperties;
+  displayLabel;
+  description;
+  propertiesMetadata;
+  relationships;
+  constructor(opts) {
+    const {
+      stream,
+      schema,
+      keyProperties,
+      bookmarkProperties,
+      displayLabel,
+      description,
+      relationships,
+      propertiesMetadata
+    } = opts;
+    this.stream = stream;
+    this.schema = schema;
+    this.keyProperties = keyProperties;
+    this.bookmarkProperties = bookmarkProperties;
+    this.displayLabel = displayLabel;
+    this.description = description;
+    this.relationships = relationships;
+    this.propertiesMetadata = propertiesMetadata;
+  }
+  toString() {
+    const result = {
+      type: "SCHEMA",
+      stream: this.stream,
+      schema: this.schema,
+      key_properties: this.keyProperties,
+      label: this.displayLabel,
+      description: this.description,
+      relationships: this.relationships
+    };
+    if (this.bookmarkProperties) {
+      result["bookmark_properties"] = this.bookmarkProperties;
+    }
+    return JSON.stringify(result);
+  }
+};
+
 // src/sdk/models/metadata.ts
 var MetadataMap = class extends Map {
   areEquals(array1, array2) {
@@ -826,155 +826,6 @@ var StreamMetadata = class {
   }
 };
 
-// src/sdk/models/catalog.ts
-var _ = __toESM(require("lodash"));
-var Catalog = class _Catalog {
-  streams;
-  static fromFile(fileName) {
-    const catalogFile = loadJson(fileName);
-    const catalog = new _Catalog();
-    catalog.streams = catalogFile.streams;
-    return catalog;
-  }
-  static fromCatalogFile(catalogFile) {
-    const catalog = new _Catalog();
-    catalog.streams = catalogFile.streams;
-    return catalog;
-  }
-  getSelectedStreams() {
-    if (!this.streams) {
-      return [];
-    }
-    const directSelectedStreams = this.streams.filter((stream) => {
-      const metadata = new StreamMetadata();
-      metadata.fromCatalogStreamMetadata(stream.metadata);
-      return metadata.isStreamSelected();
-    });
-    logger.debug(`Directly selected streams: ${directSelectedStreams.map((stream) => stream.stream).join(",")}`);
-    const getIndirectSelectedStreams = (stream, ancestorsIds) => {
-      const currAncestorsIds = ancestorsIds || [];
-      const rootBreadcrumbMetadata = stream.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-      const parentStreamId = rootBreadcrumbMetadata?.metadata?.["whaly-parent-stream"];
-      if (parentStreamId) {
-        const parentStream = this.streams?.find((stream2) => stream2.stream === parentStreamId);
-        if (parentStream) {
-          currAncestorsIds.push(parentStreamId);
-          return getIndirectSelectedStreams(parentStream, currAncestorsIds);
-        }
-      }
-      return currAncestorsIds;
-    };
-    const indirectSelectedStreamIds = _.uniq(
-      directSelectedStreams.flatMap((stream) => {
-        return getIndirectSelectedStreams(stream);
-      })
-    );
-    logger.debug(`Indirectly selected streams: ${indirectSelectedStreamIds.join(",")}`);
-    const indirectSelectedStreams = indirectSelectedStreamIds.map((streamId) => {
-      if (!directSelectedStreams.find((stream) => stream.stream === streamId)) {
-        return this.streams?.find((stream) => stream.stream === streamId);
-      }
-    }).filter((stream) => !!stream);
-    return directSelectedStreams.concat(indirectSelectedStreams);
-  }
-};
-
-// src/sdk/models/messages.ts
-var RecordMessage = class {
-  type = "RECORD";
-  stream;
-  record;
-  constructor(record, stream) {
-    this.record = record;
-    this.stream = stream;
-  }
-  toString() {
-    const result = {
-      type: "RECORD",
-      record: this.record,
-      stream: this.stream
-    };
-    return JSON.stringify(result);
-  }
-};
-var StateMessage = class {
-  type = "STATE";
-  value;
-  constructor(value) {
-    this.value = value;
-  }
-  toString() {
-    const result = {
-      type: "STATE",
-      value: this.value
-    };
-    return JSON.stringify(result);
-  }
-};
-var ReplicationMethodMessage = class {
-  type = "REPLICATION_METHOD";
-  stream;
-  replication;
-  constructor(stream, version) {
-    this.stream = stream;
-    this.replication = version;
-  }
-  toString() {
-    const result = {
-      type: "REPLICATION_METHOD",
-      replication: this.replication,
-      stream: this.stream
-    };
-    return JSON.stringify(result);
-  }
-};
-var SchemaMessage = class {
-  type = "SCHEMA";
-  stream;
-  schema;
-  keyProperties;
-  bookmarkProperties;
-  displayLabel;
-  description;
-  propertiesMetadata;
-  relationships;
-  constructor(opts) {
-    const {
-      stream,
-      schema,
-      keyProperties,
-      bookmarkProperties,
-      displayLabel,
-      description,
-      relationships,
-      propertiesMetadata
-    } = opts;
-    this.stream = stream;
-    this.schema = schema;
-    this.keyProperties = keyProperties;
-    this.bookmarkProperties = bookmarkProperties;
-    this.displayLabel = displayLabel;
-    this.description = description;
-    this.relationships = relationships;
-    this.propertiesMetadata = propertiesMetadata;
-  }
-  toString() {
-    const result = {
-      type: "SCHEMA",
-      stream: this.stream,
-      schema: this.schema,
-      key_properties: this.keyProperties,
-      label: this.displayLabel,
-      description: this.description,
-      relationships: this.relationships
-    };
-    if (this.bookmarkProperties) {
-      result["bookmark_properties"] = this.bookmarkProperties;
-    }
-    return JSON.stringify(result);
-  }
-};
-
 // src/sdk/models/state.ts
 var import_moment = __toESM(require("moment"));
 
@@ -1155,186 +1006,46 @@ var isDatetimeType = (typeDef) => {
 };
 
 // src/sdk/models/tap/stream.ts
-var _2 = __toESM(require("lodash"));
-var import_bluebird2 = require("bluebird");
+var import_cloneDeep = __toESM(require("lodash/cloneDeep.js"));
+var import_bluebird2 = __toESM(require("bluebird"));
 
 // src/sdk/models/tap/tap.ts
-var import_bluebird = require("bluebird");
-var import_dependency_graph = require("dependency-graph");
+var import_bluebird = __toESM(require("bluebird"));
 var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
 var Tap = class {
-  _target;
-  _config;
-  _streams;
-  _concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
+  target;
+  config;
+  streams;
+  concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
   constructor(target, config) {
-    this._streams = [];
-    this._target = target;
-    this._config = config;
-  }
-  discover = async (existingCatalog) => {
-    logger.debug("Loading schemas");
-    const streamDepGraph = new import_dependency_graph.DepGraph();
-    const buildDependencyGraph = (streams2, parentId) => {
-      streams2.forEach((s) => {
-        streamDepGraph.addNode(s.streamId, s);
-        if (parentId) {
-          streamDepGraph.addDependency(s.streamId, parentId);
-        }
-        if (s.children) {
-          return buildDependencyGraph(s.children, s.streamId);
-        }
-      });
-    };
-    buildDependencyGraph(this._streams);
-    const discoveredStreamCatalogEntries = await import_bluebird.Promise.map(
-      streamDepGraph.overallOrder(),
-      async (streamId) => {
-        logger.info(`\u2728 Loading schema for ${streamId}`);
-        const stream = streamDepGraph.getNodeData(streamId);
-        const schema = await stream.getSchema();
-        const parentStreamIds = streamDepGraph.directDependenciesOf(streamId);
-        if (parentStreamIds.length > 1) {
-          throw new Error(`StreamId=${streamId} directly depends on more than 1 stream, e.g. ${parentStreamIds.join(",")} which is not supported!`);
-        }
-        const parentStreamId = parentStreamIds?.[0];
-        const { metadata } = await this.generateMetadataFromStream(stream, parentStreamId);
-        return {
-          schema,
-          metadata,
-          stream: stream.streamId,
-          tap_stream_id: stream.streamId
-        };
-      },
-      { concurrency: this._concurrency }
-    );
-    const existingCatalogStreamIds = (existingCatalog?.streams || []).map((entry) => {
-      return entry.stream;
-    });
-    const newlyDiscoveredStreamCatalogEntries = discoveredStreamCatalogEntries.filter((entry) => {
-      if (existingCatalogStreamIds.includes(entry.stream)) {
-        return false;
-      } else {
-        return true;
-      }
-    });
-    logger.debug(`Newly discovered streams: %j`, newlyDiscoveredStreamCatalogEntries);
-    const allDiscoveredStreamIds = discoveredStreamCatalogEntries.map((entry) => {
-      return entry.stream;
-    });
-    const existingCatalogMinusDeletedStreams = (existingCatalog?.streams || []).filter((entry) => {
-      if (!allDiscoveredStreamIds.includes(entry.stream)) {
-        logger.debug(`Stream %s was deleted, removing it from the catalog`, entry.stream);
-        return false;
-      } else {
-        return true;
-      }
-    });
-    const updatedExistingCatalog = existingCatalogMinusDeletedStreams.map((existingCatalogEntry) => {
-      const matchingDiscoveredCatalogEntry = discoveredStreamCatalogEntries.find((entry) => entry.stream === existingCatalogEntry.stream);
-      if (matchingDiscoveredCatalogEntry) {
-        const existingRootMetadata = existingCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-        const discoveredRootMetadata = matchingDiscoveredCatalogEntry.metadata.find((metadata) => metadata.breadcrumb.length === 0);
-        if (existingRootMetadata) {
-          const parentStreamVal = discoveredRootMetadata?.metadata["whaly-parent-stream"];
-          if (parentStreamVal !== void 0) {
-            existingRootMetadata.metadata["whaly-parent-stream"] = parentStreamVal;
-          }
-          const forcedReplicationMethodVal = discoveredRootMetadata?.metadata["forced-replication-method"];
-          if (forcedReplicationMethodVal !== void 0) {
-            existingRootMetadata.metadata["forced-replication-method"] = forcedReplicationMethodVal;
-          }
-          const forcedReplicationKeyVal = discoveredRootMetadata?.metadata["forced-replication-key"];
-          if (forcedReplicationKeyVal !== void 0) {
-            existingRootMetadata.metadata["forced-replication-key"] = forcedReplicationKeyVal;
-          }
-          const validReplicationKeysVal = discoveredRootMetadata?.metadata["valid-replication-keys"];
-          if (validReplicationKeysVal !== void 0) {
-            existingRootMetadata.metadata["valid-replication-keys"] = validReplicationKeysVal;
-          }
-          const rowCountVal = discoveredRootMetadata?.metadata["row-count"];
-          if (rowCountVal !== void 0) {
-            existingRootMetadata.metadata["row-count"] = rowCountVal;
-          }
-        }
-      }
-      return existingCatalogEntry;
-    });
-    const streams = updatedExistingCatalog.concat(newlyDiscoveredStreamCatalogEntries);
-    return { streams };
-  };
-  sync = async (catalog) => {
-    await this.initiateSync(catalog);
-    logger.debug(`[TAP] All streams have finished syncing. We send the [complete] signal to the target.`);
-    await this._target.complete();
-    return Promise.resolve();
-  };
-  async generateMetadataFromStream(stream, parentStreamId) {
-    const metadata = new StreamMetadata();
-    metadata.write([], "table-key-properties", stream.primaryKey);
-    const defaultReplicationConfig = stream.defaultReplicationConfig();
-    metadata.write([], "replication-method", defaultReplicationConfig.replicationMethod);
-    const validReplicationkeys = await stream.getValidReplicationKeys();
-    metadata.write([], "valid-replication-keys", validReplicationkeys);
-    if (defaultReplicationConfig.isForced) {
-      metadata.write([], "forced-replication-method", defaultReplicationConfig.replicationMethod);
-      if (stream.forcedReplicationKey) {
-        metadata.write([], "forced-replication-key", stream.forcedReplicationKey);
-      }
-    }
-    const rowCount = await stream.getRowCount();
-    if (rowCount !== void 0) {
-      metadata.write([], "row-count", rowCount);
-    }
-    if (stream.isSilent === true) {
-      metadata.write([], "whaly-is-hidden", stream.isSilent);
-    }
-    if (stream.isSilent === false) {
-      metadata.write([], "selected-by-default", stream.selectedByDefault);
-    }
-    if (stream.displayLabel) {
-      metadata.write([], "whaly-display-label", stream.displayLabel);
-    }
-    if (stream.description) {
-      metadata.write([], "whaly-description", stream.description);
-    }
-    if (parentStreamId !== void 0) {
-      metadata.write([], "whaly-parent-stream", parentStreamId);
-    }
-    return { metadata: metadata.toList() };
+    this.streams = [];
+    this.target = target;
+    this.config = config;
   }
-  async initiateSync(catalog) {
+  sync = async (options) => {
     logger.info(`\u{1F680} Start syncing`);
     const state = StateService.getInstance().get();
     logger.info(`\u{1F4CD} Received state: %j`, state);
-    logger.debug(`\u{1F4DA} Received catalog: %j`, catalog);
-    const selectedStreams = catalog.getSelectedStreams();
-    if (selectedStreams.length === 0) {
-      logger.info(`\u{1F631} There is no stream to sync. 
-            
-            Did you update the \`catalog.json\` to add \`"selected": true\` in the table metadata associated with the root breadcrumb (e.g. \`[]\`)`);
-      return;
-    }
-    const selectedStreamIds = selectedStreams.map((stream) => stream.stream);
-    logger.info(`\u{1F973} Syncing selected streams: %s`, selectedStreamIds.join(","));
-    await import_bluebird.Promise.map(
-      this._streams,
-      (stream) => {
-        if (selectedStreamIds.includes(stream.streamId)) {
-          const selectedStream = selectedStreams.find(((selectedStream2) => selectedStream2.stream === stream.streamId));
-          const rootMetadataEntry = selectedStream?.metadata.find((streamMdt) => streamMdt.breadcrumb.length === 0);
-          if (!rootMetadataEntry) {
-            throw new Error(`Can't find a catalog root metadata entry for StreamId=${stream.streamId}`);
-          }
-          stream.setRootMetadataEntry(rootMetadataEntry.metadata);
-          return stream.sync();
-        }
-      },
-      {
-        concurrency: this._concurrency
-      }
-    );
-  }
+    const envInclude = process.env.TAP_STREAMS ? process.env.TAP_STREAMS.split(",").map((s) => s.trim()).filter((s) => s.length > 0) : void 0;
+    const include = options?.include && options.include.length > 0 ? options.include : envInclude;
+    const exclude = new Set((options?.exclude || []).map((s) => s.trim()));
+    let streamsToRun = this.streams.filter((s) => !s.isSilent);
+    if (include && include.length > 0) {
+      const includeSet = new Set(include);
+      streamsToRun = streamsToRun.filter((s) => includeSet.has(s.streamId));
+    }
+    if (exclude.size > 0) {
+      streamsToRun = streamsToRun.filter((s) => !exclude.has(s.streamId));
+    }
+    const selectedStreamIds = streamsToRun.map((s) => s.streamId);
+    logger.info(`\u{1F973} Syncing streams: %s`, selectedStreamIds.join(","));
+    await import_bluebird.default.map(streamsToRun, (stream) => {
+      return stream.sync();
+    }, { concurrency: this.concurrency });
+    logger.debug(`[TAP] All streams have finished syncing. We send the [complete] signal to the target.`);
+    await this.target.complete();
+    return Promise.resolve();
+  };
   // Used to clean up any ressource before exit
   end() {
     return Promise.resolve();
@@ -1343,15 +1054,11 @@ var Tap = class {
 
 // src/sdk/models/tap/stream.ts
 var import_util2 = require("util");
-var StreamV2 = class {
+var Stream = class {
   // Runtime values, can't be overriden
-  _config;
-  _tapName = "default";
-  _tapState;
-  _target;
-  // Contains the root metadata entry of the catalog stream configuration
-  // contains some end user schema configuration
-  rootMetadataEntry;
+  config;
+  tapState;
+  target;
   // Compile time values, can be overriden
   // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
   forcedReplicationMethod = void 0;
@@ -1377,11 +1084,10 @@ var StreamV2 = class {
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
-  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
-    this._tapName = tapName;
-    this._config = config;
-    this._tapState = _2.cloneDeep(state);
-    this._target = target;
+  constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    this.config = config;
+    this.tapState = (0, import_cloneDeep.default)(state);
+    this.target = target;
     this.childConcurrency = childConcurrency;
     this.rowsSyncedMetricsConf = {
       isEnabled: () => !this.isSilent,
@@ -1392,9 +1098,7 @@ var StreamV2 = class {
       getStreamIds: () => [this.streamId]
     };
   }
-  setRootMetadataEntry = (rootMetadataEntry) => {
-    this.rootMetadataEntry = rootMetadataEntry;
-  };
+  // No root metadata: replication config is derived from stream defaults only
   /**
    * Sync this stream
    * Called only for streams with no parents, aka. "root tap streams"
@@ -1425,7 +1129,7 @@ var StreamV2 = class {
       `\u{1F6BD} Flushing ${replicationMethod} sync of stream '${this.streamId}' + all of its children...`
     );
     await this._flush();
-    await import_bluebird2.Promise.map(this.children, (child) => {
+    await import_bluebird2.default.map(this.children, (child) => {
       return child.flush();
     }, { concurrency: 3 });
   };
@@ -1435,7 +1139,7 @@ var StreamV2 = class {
    */
   _writeStateMessage = () => {
     const message = new StateMessage(StateService.getInstance().get());
-    return this._target.state(message);
+    return this.target.state(message);
   };
   /**
    * Write out a ACTIVATE_VERSION message.
@@ -1443,10 +1147,10 @@ var StreamV2 = class {
   _writeReplicationMethodMessage = async () => {
     const method = this.configuredReplicationConfig().replicationMethod;
     const message = new ReplicationMethodMessage(this.streamId, method);
-    await import_bluebird2.Promise.map(this.children, async (c) => {
+    await import_bluebird2.default.map(this.children, async (c) => {
       return c._writeReplicationMethodMessage();
     }, { concurrency: 3 });
-    await this._target.replicationMethod(message);
+    await this.target.replicationMethod(message);
   };
   /**
    * Write out a SCHEMA message with the stream schema.
@@ -1467,11 +1171,11 @@ var StreamV2 = class {
           ...this.description ? { description: this.description } : {},
           ...schema.propertiesMetadata ? { propertiesMetadata: schema.propertiesMetadata } : {}
         });
-        await this._target.schema(message);
+        await this.target.schema(message);
       }
     }
     if (skipChildrenSchema !== true) {
-      await import_bluebird2.Promise.map(this.children, async (child) => {
+      await import_bluebird2.default.map(this.children, async (child) => {
         await child._writeSchemaMessage();
       }, { concurrency: 3 });
     }
@@ -1546,8 +1250,8 @@ var StreamV2 = class {
    **/
   getStartingTimestamp() {
     const streamIdToReadFrom = this.useStateFromStreamId ? this.useStateFromStreamId : this.streamId;
-    const state = extractStateForStream(this._tapState, streamIdToReadFrom);
-    const startDate = this._config.start_date;
+    const state = extractStateForStream(this.tapState, streamIdToReadFrom);
+    const startDate = this.config.start_date;
     const replicationConfig = this.configuredReplicationConfig();
     logger.debug(`${this.streamId}: state.replicationKeyValue: ${state.replicationKeyValue}, 
         state.replicationKey: ${state.replicationKey},
@@ -1606,16 +1310,10 @@ var StreamV2 = class {
    * Return the configured replication method that will be used for the sync.
    */
   configuredReplicationConfig = () => {
-    const defaultReplicationMethod = this.defaultReplicationConfig();
-    if (defaultReplicationMethod.isForced) {
-      return {
-        replicationMethod: defaultReplicationMethod.replicationMethod,
-        replicationKey: defaultReplicationMethod.replicationKey
-      };
-    }
+    const defaults = this.defaultReplicationConfig();
     return {
-      replicationMethod: this.rootMetadataEntry?.["replication-method"] || defaultReplicationMethod.replicationMethod,
-      replicationKey: this.rootMetadataEntry?.["replication-key"] || defaultReplicationMethod.replicationKey
+      replicationMethod: defaults.replicationMethod,
+      replicationKey: defaults.replicationKey
     };
   };
   // Private sync methods:
@@ -1645,9 +1343,9 @@ var StreamV2 = class {
         this.streamId
       );
       if (!this.isSilent) {
-        await this._target.record(recordMessage);
+        await this.target.record(recordMessage);
       }
-      await import_bluebird2.Promise.map(
+      await import_bluebird2.default.map(
         this.children,
         async (child) => {
           await child.asyncInit(row);
@@ -1691,7 +1389,7 @@ var StreamV2 = class {
   getSchema() {
     logger.debug(`stream: ${this.streamId} - getSchema() is called`);
     if (this.schemaPath) {
-      const schema = loadJson(`schemas/${this._tapName}/${this.schemaPath}`);
+      const schema = loadJson(`schemas/${this.schemaPath}`);
       return Promise.resolve({ jsonSchema: schema });
     } else {
       if (this.isSilent) {
@@ -1728,7 +1426,7 @@ var StreamV2 = class {
 };
 
 // src/sdk/helpers/qs-logger.ts
-var _3 = __toESM(require("lodash"));
+var import_isPlainObject = __toESM(require("lodash/isPlainObject.js"));
 var MAX_QS_KEYS = 10;
 var MAX_QS_STRING_LENGTH = 200;
 var MAX_QS_ARRAY_ITEMS = 5;
@@ -1749,7 +1447,7 @@ function truncateQueryParams(value, depth = 0) {
     }
     return items;
   }
-  if (_3.isPlainObject(value)) {
+  if ((0, import_isPlainObject.default)(value)) {
     const keys = Object.keys(value);
     const result = {};
     for (const key of keys.slice(0, MAX_QS_KEYS)) {
@@ -1772,11 +1470,12 @@ function getTruncatedParamsForLog(params) {
 }
 
 // src/sdk/models/tap/restStream.ts
-var _4 = __toESM(require("lodash"));
-var RESTStreamV2 = class extends StreamV2 {
+var import_cloneDeep2 = __toESM(require("lodash/cloneDeep.js"));
+var import_isEqual = __toESM(require("lodash/isEqual.js"));
+var RESTStream = class extends Stream {
   axiosInstance;
-  constructor(tapName, config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
-    super(tapName, config, state, target, childConcurrency);
+  constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+    super(config, state, target, childConcurrency);
     this.apiCallsMetricsConf = {
       isEnabled: () => !this.isSilent,
       getStreamIds: () => [this.streamId]
@@ -1799,7 +1498,7 @@ var RESTStreamV2 = class extends StreamV2 {
     const authenticator = this.authenticator;
     const params = {
       ...this.getNextUrlParams(nextPageToken, parent),
-      ...await authenticator?.getAuthQS(this._config)
+      ...await authenticator?.getAuthQS(this.config)
     };
     const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
     let headers = this.getCustomHTTPHeaders(parent);
@@ -1807,7 +1506,7 @@ var RESTStreamV2 = class extends StreamV2 {
       headers = {};
     }
     if (authenticator) {
-      const authHeaders = await authenticator.getAuthHeaders(this._config);
+      const authHeaders = await authenticator.getAuthHeaders(this.config);
       headers = {
         ...headers,
         ...authHeaders
@@ -1890,9 +1589,9 @@ var RESTStreamV2 = class extends StreamV2 {
       for await (const row of rows) {
         yield row;
       }
-      const previousToken = _4.cloneDeep(nextPageToken);
+      const previousToken = (0, import_cloneDeep2.default)(nextPageToken);
       nextPageToken = this.getNextPageToken(resp, previousToken);
-      if (nextPageToken && _4.isEqual(nextPageToken, previousToken)) {
+      if (nextPageToken && (0, import_isEqual.default)(nextPageToken, previousToken)) {
         throw new Error(
           `${this.streamId} - Loop detected in pagination. "
                     Pagination token: \`${JSON.stringify(nextPageToken)}\` is identical to prior token \`${JSON.stringify(previousToken)}\`.`
@@ -2093,20 +1792,20 @@ var StreamWarehouseSyncService = class {
     const columnNamesFromWarehouse = columnsFromWarehouse.map((col) => col.name);
     const columnsMappingStore = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
     const columnsToAdd = Object.keys(columnsMappingStore).filter((unsafeColumnKey) => {
-      const safeColumnName = columnsMappingStore[unsafeColumnKey];
-      if (!safeColumnName) {
+      const safeColumnName2 = columnsMappingStore[unsafeColumnKey];
+      if (!safeColumnName2) {
         return false;
       }
-      return !columnNamesFromWarehouse.includes(safeColumnName);
+      return !columnNamesFromWarehouse.includes(safeColumnName2);
     }).map((unsafeColumnKey) => {
-      const safeColumnName = columnsMappingStore[unsafeColumnKey];
+      const safeColumnName2 = columnsMappingStore[unsafeColumnKey];
       const definition = this.flattenedSchema[unsafeColumnKey];
-      if (!safeColumnName || !definition) {
+      if (!safeColumnName2 || !definition) {
         return void 0;
       }
       return {
         unsafeName: unsafeColumnKey,
-        name: safeColumnName,
+        name: safeColumnName2,
         definition
       };
     }).filter((item) => !!item);
@@ -2328,10 +2027,10 @@ var import_moment3 = __toESM(require("moment"));
 
 // src/sdk/models/target/temporaryFile.ts
 var import_fs_extra2 = __toESM(require("fs-extra"));
-var import_lodash = require("lodash");
+var import_uniqueId = __toESM(require("lodash/uniqueId.js"));
 var createTemporaryFileStream = (streamId) => {
   import_fs_extra2.default.ensureDirSync("./tmp");
-  const path = `./tmp/${safePath(streamId)}-${(0, import_lodash.uniqueId)()}.tmp`;
+  const path = `./tmp/${safePath(streamId)}-${(0, import_uniqueId.default)()}.tmp`;
   const writeStream = import_fs_extra2.default.createWriteStream(path, { flags: "w" });
   return {
     stream: writeStream,
@@ -2456,7 +2155,7 @@ var StreamState = class {
 
 // src/sdk/models/target/target.ts
 var import_async_mutex = require("async-mutex");
-var import_bluebird3 = require("bluebird");
+var import_bluebird3 = __toESM(require("bluebird"));
 var semaphore = new import_async_mutex.Semaphore(1);
 var ITarget = class _ITarget {
   config;
@@ -2701,7 +2400,7 @@ var ITarget = class _ITarget {
           })
         }
       };
-      await import_bluebird3.Promise.map(this.schemaHooks, async (hook) => {
+      await import_bluebird3.default.map(this.schemaHooks, async (hook) => {
         const input = {
           databaseName,
           schemaName,
@@ -2782,7 +2481,7 @@ var ITarget = class _ITarget {
   };
   uploadStreamsToWarehouse = async (streamsToUpload) => {
     try {
-      await import_bluebird3.Promise.map(streamsToUpload, async (streamId) => {
+      await import_bluebird3.default.map(streamsToUpload, async (streamId) => {
         logger.info(`\u{1F4E4} Upload stream: ${streamId} to Warehouse`);
         const streamState = this.streams[streamId];
         if (streamState) {
@@ -2816,20 +2515,644 @@ var ITarget = class _ITarget {
     }
   };
 };
+
+// src/targets/bigquery/helpers.ts
+var safeColumnName = (key) => {
+  let returnString = key;
+  const shouldNotStartWith = ["_TABLE_", "_FILE_", "_PARTITION"];
+  shouldNotStartWith.forEach((snm) => {
+    if (returnString.startsWith(snm)) {
+      returnString = returnString.replace(snm, "");
+    }
+  });
+  returnString = returnString.replace(/^[-\d\s]*/g, "");
+  returnString = returnString.toLowerCase();
+  returnString = returnString.replace("`", "");
+  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+  if (returnString.length > 300) {
+    returnString = returnString.substr(0, 300);
+  }
+  const pattern = /[^a-zA-Z0-9_]/gm;
+  return returnString.replace(pattern, "_");
+};
+var safeTableName = (key) => {
+  let returnString = key;
+  returnString = returnString.toLowerCase();
+  returnString = returnString.replace("`", "");
+  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+  if (returnString.length > 1024) {
+    returnString = returnString.substr(0, 1024);
+  }
+  const pattern = /[^a-zA-Z0-9_]/gm;
+  return returnString.replace(pattern, "_");
+};
+
+// src/targets/bigquery/service/record.ts
+var import_decimal = __toESM(require("decimal.js"));
+var import_moment5 = __toESM(require("moment"));
+var validateDateRange = (record, schema) => {
+  Object.keys(schema).forEach((key) => {
+    const { format: format3 } = schema[key];
+    if (format3 === "date-time") {
+      if (record[key]) {
+        const formattedDate = (0, import_moment5.default)(record[key]).format(defaultDateTimeFormat);
+        const isNotTooMuchInTheFuture = (0, import_moment5.default)(record[key]).isBefore((0, import_moment5.default)("9999-12-31 23:59:59.999999"));
+        const isNotTooMuchInThePast = (0, import_moment5.default)(record[key]).isAfter((0, import_moment5.default)("0001-01-01 00:00:00"));
+        if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
+          record[key] = formattedDate;
+        } else {
+          record[key] = void 0;
+          logger.info("Skipping date for key %s for being invalid", key);
+        }
+      }
+    }
+  });
+  return record;
+};
+var maxBQNumericValue = new import_decimal.default("9.9999999999999999999999999999999999999E+28");
+var convertNumberIntoDecimal = (record, schema) => {
+  Object.keys(record).forEach((key) => {
+    if (schema[key]) {
+      const { type, items } = schema[key];
+      const types = type instanceof Array ? type : [type];
+      if (types.includes("number")) {
+        if (typeof record[key] === "number" || record[key] instanceof import_decimal.default) {
+          const decimal = new import_decimal.default(record[key]).toDecimalPlaces(9);
+          if (decimal.greaterThanOrEqualTo(maxBQNumericValue)) {
+            record[key] = null;
+          } else {
+            record[key] = decimal;
+          }
+        } else {
+          record[key] = null;
+        }
+      } else if (types.includes("array") && items?.type.includes("number")) {
+        if (Array.isArray(record[key])) {
+          const rawArr = record[key];
+          const newArr = rawArr.map((item) => {
+            if (typeof item === "number" || item instanceof import_decimal.default) {
+              const parsed = new import_decimal.default(item).toDecimalPlaces(9);
+              if (parsed.greaterThanOrEqualTo(maxBQNumericValue)) {
+                return void 0;
+              } else {
+                return parsed;
+              }
+            } else {
+              return void 0;
+            }
+          }).filter((item) => !!item);
+          record[key] = newArr;
+        } else {
+          record[key] = null;
+        }
+      }
+    }
+  });
+  return record;
+};
+
+// src/targets/bigquery/service/dbSync.ts
+var import_moment6 = __toESM(require("moment"));
+
+// src/targets/bigquery/service/bigquery.ts
+var import_bigquery = require("@google-cloud/bigquery");
+var BqClientHolder = class {
+  static instance;
+  client;
+  constructor(config) {
+    const retryOptions = {
+      autoRetry: true,
+      maxRetries: 3
+    };
+    this.client = new import_bigquery.BigQuery({
+      projectId: config.database.trim(),
+      ...retryOptions
+    });
+  }
+  static client(config) {
+    if (!this.instance) {
+      this.instance = new this(config);
+    }
+    return this.instance.client;
+  }
+};
+
+// src/targets/bigquery/service/cloudStorage.ts
+var import_storage = require("@google-cloud/storage");
+
+// node_modules/uuid/dist/esm-node/rng.js
+var import_crypto = __toESM(require("crypto"));
+var rnds8Pool = new Uint8Array(256);
+var poolPtr = rnds8Pool.length;
+function rng() {
+  if (poolPtr > rnds8Pool.length - 16) {
+    import_crypto.default.randomFillSync(rnds8Pool);
+    poolPtr = 0;
+  }
+  return rnds8Pool.slice(poolPtr, poolPtr += 16);
+}
+
+// node_modules/uuid/dist/esm-node/regex.js
+var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
+
+// node_modules/uuid/dist/esm-node/validate.js
+function validate(uuid) {
+  return typeof uuid === "string" && regex_default.test(uuid);
+}
+var validate_default = validate;
+
+// node_modules/uuid/dist/esm-node/stringify.js
+var byteToHex = [];
+for (let i = 0; i < 256; ++i) {
+  byteToHex.push((i + 256).toString(16).substr(1));
+}
+function stringify3(arr, offset = 0) {
+  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
+  if (!validate_default(uuid)) {
+    throw TypeError("Stringified UUID is invalid");
+  }
+  return uuid;
+}
+var stringify_default = stringify3;
+
+// node_modules/uuid/dist/esm-node/v4.js
+function v4(options, buf, offset) {
+  options = options || {};
+  const rnds = options.random || (options.rng || rng)();
+  rnds[6] = rnds[6] & 15 | 64;
+  rnds[8] = rnds[8] & 63 | 128;
+  if (buf) {
+    offset = offset || 0;
+    for (let i = 0; i < 16; ++i) {
+      buf[offset + i] = rnds[i];
+    }
+    return buf;
+  }
+  return stringify_default(rnds);
+}
+var v4_default = v4;
+
+// src/targets/bigquery/service/cloudStorage.ts
+var uploadFileInBucket = async (streamId, gcsBuckerName, filePath) => {
+  try {
+    const retryOptions = { autoRetry: true, maxRetries: 20 };
+    let storage = new import_storage.Storage({ retryOptions });
+    const bucketRef = storage.bucket(gcsBuckerName);
+    await bucketRef.get({ autoCreate: false });
+    const destinationFileName = `${streamId}-${v4_default()}.jsonnl`;
+    await bucketRef.upload(filePath, {
+      destination: destinationFileName
+    });
+    logger.info(`\u{1F5F3}  Uploaded ${filePath} into ${gcsBuckerName}/${destinationFileName} GCS File`);
+    return bucketRef.file(destinationFileName);
+  } catch (err) {
+    logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
+        
+        Error: ${err.message}
+        Stack: ${err.stack}
+        Code: ${err.code}
+        `);
+    if (err.code < 500) {
+      await haltAndCatchFire(
+        `unauthorized`,
+        `We couldn't connect to your Cloud Storage bucket \u{1F614}
+                    
+                    The error from Google is: '${err.message}' \u{1F440}
+                    
+                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
+        `Got error from cloud storage lib`
+      );
+    }
+    throw err;
+  }
+};
+
+// src/targets/bigquery/service/dbSync.ts
+var import_chunk = __toESM(require("lodash/chunk.js"));
+var import_async_mutex2 = require("async-mutex");
+var semaphore2 = new import_async_mutex2.Semaphore(1);
+var BigQueryDBSync = class extends StreamWarehouseSyncService {
+  config;
+  bqClient;
+  bqDataset;
+  bqTable;
+  bqStagingTable;
+  tableName;
+  stagingTableName;
+  sqlTableId;
+  sqlStagingTableId;
+  constructor(config, streamId, database, schema, table, renameColumnStore) {
+    super(
+      streamId,
+      database,
+      schema,
+      table,
+      renameColumnStore
+    );
+    this.config = config;
+    this.bqClient = BqClientHolder.client(config);
+    this.bqDataset = this.bqClient.dataset(schema, { location: config.location || "EU" });
+    this.bqTable = this.bqDataset.table(table, { location: config.location || "EU" });
+    this.tableName = table;
+    this.stagingTableName = `${table}_temp`;
+    this.bqStagingTable = this.bqDataset.table(this.stagingTableName, { location: config.location || "EU" });
+    this.sqlTableId = `${schema}.${table}`;
+    this.sqlStagingTableId = `${this.sqlTableId}_temp`;
+  }
+  // Implementation of abstract classes
+  getSerializedRecord = (record) => {
+    return JSON.stringify(record) + `
+`;
+  };
+  safeColumnName = safeColumnName;
+  getWarehouseTypeFromJSONSchema = (definition) => {
+    const type = definition.type;
+    const format3 = definition.format;
+    let typeArr;
+    if (type instanceof Array) {
+      typeArr = type;
+    } else {
+      typeArr = [type];
+    }
+    if (format3) {
+      switch (format3) {
+        case "date-time":
+          return "TIMESTAMP";
+        case "json":
+          return "STRING";
+        default:
+          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format3}`);
+      }
+    } else if (typeArr.includes("number")) {
+      return "NUMERIC";
+    } else if (typeArr.includes("integer") && type.includes("string")) {
+      return "NUMERIC";
+    } else if (typeArr.includes("integer")) {
+      return "INTEGER";
+    } else if (typeArr.includes("boolean")) {
+      return "BOOLEAN";
+    } else if (typeArr.includes("string")) {
+      return "STRING";
+    } else if (typeArr.includes("array")) {
+      const items = definition.items;
+      if (items) {
+        return this.getWarehouseTypeFromJSONSchema(items);
+      }
+      throw new Error(`StreamId: ${this.streamId} -  Unsupported array schema field definition: ${JSON.stringify(definition)}`);
+    } else {
+      throw new Error(`StreamId: ${this.streamId} -  Unsupported schema field definition: ${JSON.stringify(definition)}`);
+    }
+  };
+  getMergeQueries = () => {
+    const primaryKeyCondition = this.primaryKeys.map((pk) => {
+      return `\`sourced\`.\`${pk}\` = \`destination\`.\`${pk}\``;
+    }).join(` AND `);
+    const primaryKeys = this.primaryKeys.map((pk) => `\`${pk}\``).join(` ,`);
+    const columnUnsafeToSafeMapping = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    const columns = Object.keys(columnUnsafeToSafeMapping).map((column) => {
+      return {
+        isUnsafe: true,
+        colName: column
+      };
+    });
+    const columnsChunks = (0, import_chunk.default)(columns, 300).map((chunk2) => {
+      this.primaryKeys.forEach((pk) => {
+        if (!chunk2.map((c) => columnUnsafeToSafeMapping[c.colName]).includes(pk)) {
+          chunk2.push({
+            isUnsafe: false,
+            colName: pk
+          });
+        }
+      });
+      return chunk2;
+    });
+    return columnsChunks.map((chunk2) => {
+      const setValues = chunk2.map((column) => {
+        if (column.isUnsafe) {
+          return `\`destination\`.\`${columnUnsafeToSafeMapping[column.colName]}\` = \`sourced\`.\`${columnUnsafeToSafeMapping[column.colName]}\``;
+        } else {
+          return `\`destination\`.\`${column.colName}\` = \`sourced\`.\`${column.colName}\``;
+        }
+      }).join(`, `);
+      const renamedCols = chunk2.map((column) => {
+        if (column.isUnsafe) {
+          return `\`${columnUnsafeToSafeMapping[column.colName]}\``;
+        } else {
+          return `\`${column.colName}\``;
+        }
+      }).join(`, `);
+      const query = `
+        MERGE ${this.sqlTableId} destination
+        USING (
+            WITH all_staging_rows AS (
+                SELECT *
+                FROM ${this.sqlStagingTableId}
+            ),
+            numbered_rows AS (
+                SELECT *, 
+                    ROW_NUMBER() OVER (PARTITION BY ${primaryKeys}) as _wly_row_nb,
+                    COUNT(1) OVER (PARTITION BY ${primaryKeys}) AS _wly_partition_size,
+                FROM all_staging_rows
+            ),
+            staging_deduped AS (
+                SELECT * EXCEPT(_wly_partition_size, _wly_row_nb)
+                FROM numbered_rows 
+                WHERE _wly_partition_size = _wly_row_nb
+            )
+            SELECT *
+            FROM staging_deduped
+        ) AS sourced
+        ON ${primaryKeyCondition}
+        WHEN MATCHED THEN
+            UPDATE SET ${setValues}
+        WHEN NOT MATCHED THEN
+            INSERT (${renamedCols}) VALUES (${renamedCols});
+    `;
+      return query;
+    });
+  };
+  getReplaceQueries = () => {
+    return [
+      `TRUNCATE TABLE ${this.sqlTableId};`,
+      ...this.getMergeQueries()
+    ];
+  };
+  createDatabaseAndSchemaIfNotExists = async (retryCount) => {
+    try {
+      await semaphore2.runExclusive(async () => {
+        const [exists] = await this.bqDataset.exists();
+        if (!exists) {
+          logger.info(`\u{1F423} Creating dataset: ${this.bqDataset.id} on BigQuery`);
+          await this.bqDataset.create();
+        } else {
+          const [metadata] = await this.bqDataset.getMetadata();
+          const alreadyExistinglocation = metadata.location;
+          if (this.config.location && this.config.location !== alreadyExistinglocation) {
+            await haltAndCatchFire(
+              `unknown`,
+              `A BigQuery dataset called '${this.bqDataset.id}' already exists in location: '${alreadyExistinglocation}' \u{1F605}
+                            
+                            This connector is configured to load data into the location '${this.config.location}' which is not possible as 2 datasets sharing the same name can't exist in different location on Google Cloud \u{1F63F}
+
+                            To fix this issue, could you either: a. Delete the already existing dataset? b. Change the configured destination schema name to avoid the name conflict? \u{1F64F}
+                            
+                            Please relaunch the connector once it's done to fix the issue \u{1F917}
+                            `,
+              `
+                            We have a conflict between an already existing dataset in location ${alreadyExistinglocation} and the configured location: ${this.config.location}
+                            `
+            );
+            throw Error();
+          }
+          logger.info(`\u{1F44B} Dataset: ${this.bqDataset.id} already exists on BigQuery, reusing it.`);
+        }
+      });
+    } catch (err) {
+      if (err.code === 400) {
+        await haltAndCatchFire(
+          `unauthorized`,
+          `We can't connect to your BigQuery account \u{1F614}
+
+                    Here is the message from BigQuery: ${err.message}
+                    
+                    Can you check that your Warehouse credentials and Source target schema are properly configured and try to sync again the source? \u{1F64F}
+                    `,
+          `Got error when trying to get Dataset from BigQuery: ${err.message} - ${err.code}`
+        );
+        return;
+      }
+      if (retryCount && retryCount > this.maxRetryCount) {
+        logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. ${err.message} - ${err.code}`);
+        gracefulExit(1);
+      } else {
+        logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. Retrying... ${err.message} - ${err.code}`);
+        await new Promise((resolve, reject) => setTimeout(resolve, 1e3 * retryCount));
+        await this.createDatabaseAndSchemaIfNotExists(retryCount + 1);
+      }
+    }
+    return Promise.resolve();
+  };
+  createTable = async () => {
+    try {
+      const tableName = this.tableName;
+      const schema = this.generateBigquerySchema();
+      logger.info(`\u{1F423} Creating table: ${tableName} on BigQuery`);
+      const createTableOptions = {
+        schema
+      };
+      await this.bqDataset.createTable(
+        tableName,
+        createTableOptions
+      );
+      return;
+    } catch (err) {
+      throw new Error(`StreamId: ${this.streamId} - Issue when creating table in BigQuery
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+  addColumns = async (tableFields) => {
+    if (tableFields.length > 0) {
+      logger.info(`\u{1F195} Stream: ${this.streamId} - Adding following columns in BigQuery schema: ${JSON.stringify(tableFields)}`);
+      const [metadata] = await this.bqTable.getMetadata();
+      const bgTableFields = tableFields.map((tblField) => {
+        return this.getBigqueryTableFieldFromInternalSchemaFieldDefinition(tblField.name, tblField.definition);
+      }).filter((fld) => {
+        return !!fld;
+      });
+      const schema = metadata.schema;
+      const new_schema = {
+        ...schema,
+        fields: schema.fields.concat(bgTableFields)
+      };
+      metadata.schema = new_schema;
+      await this.bqTable.setMetadata(metadata);
+      return;
+    }
+  };
+  createStagingArea = async () => {
+    try {
+      const tableName = this.stagingTableName;
+      const [exists] = await this.bqDataset.table(tableName).exists();
+      if (exists) {
+        logger.info(`\u{1F474} Temporary table ${tableName} already exists, it should be a relic form the past.
+                Dropping it.`);
+        await this.deleteStagingArea();
+      }
+      const schema = this.generateBigquerySchema();
+      const expirationTime = (0, import_moment6.default)().add("1", "day").valueOf().toString();
+      const createTableOptions = {
+        schema,
+        expirationTime
+      };
+      logger.info(`\u{1F423} Creating staging table: ${tableName} on BigQuery`);
+      await this.bqDataset.createTable(
+        tableName,
+        createTableOptions
+      );
+      return;
+    } catch (err) {
+      throw new Error(`StreamId: ${this.streamId} - Issue when creating table in BigQuery
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+  loadStreamInStagingArea = async (localFilePath) => {
+    try {
+      const file = await uploadFileInBucket(
+        this.streamId,
+        this.config.loading_deck_gcs_bucket_name,
+        localFilePath
+      );
+      const metadata = {
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        writeDisposition: "WRITE_TRUNCATE"
+      };
+      await this.loadGCSFileInTable(this.sqlStagingTableId, file, metadata);
+    } catch (err) {
+      logger.error(`StreamId: ${this.streamId} - Error while uploading stream.`);
+      throw err;
+    }
+  };
+  deleteStagingArea = () => {
+    const query = `DROP TABLE IF EXISTS ${this.sqlStagingTableId};`;
+    return this.runQueriesWithRetry([query]);
+  };
+  getTablesInSchema = async () => {
+    const [bqTables] = await this.bqDataset.getTables();
+    const foundTables = bqTables.filter((tbl) => {
+      return !!tbl.id;
+    }).map((bqTable) => {
+      return {
+        tableName: bqTable.id
+      };
+    });
+    return foundTables;
+  };
+  getTableColumnsFromWarehouse = async () => {
+    const [table] = await this.bqTable.getMetadata();
+    return table.schema.fields;
+  };
+  runQueries = async (queries) => {
+    const sqlQuery = `
+        BEGIN
+            ${queries.join(`
+            `)}
+        END;
+    `;
+    await this.bqClient.query(sqlQuery);
+  };
+  // Specific methods
+  /**
+   * 
+   * Generate a TableField definition object that can be passed to BigQuery APIs to define a column on a table
+   * 
+   * @param colName 
+   * @param schemaFieldDefinition 
+   * @returns 
+   */
+  getBigqueryTableFieldFromInternalSchemaFieldDefinition = (colName, schemaFieldDefinition) => {
+    const name = colName;
+    if (!schemaFieldDefinition) {
+      throw new Error(`StreamId: ${this.streamId} - Schema is unknown for colName=${colName}, so we can't infer the proper BigQuery definition for this column.`);
+    }
+    const type = schemaFieldDefinition.type;
+    let typeArr;
+    if (type instanceof Array) {
+      typeArr = type;
+    } else {
+      typeArr = [type];
+    }
+    if (typeArr.includes("array")) {
+      return {
+        name,
+        mode: "REPEATED",
+        type: this.getWarehouseTypeFromJSONSchema(schemaFieldDefinition)
+      };
+    } else {
+      let nullableMode = "NULLABLE";
+      return {
+        name,
+        mode: nullableMode,
+        type: this.getWarehouseTypeFromJSONSchema(schemaFieldDefinition)
+      };
+    }
+  };
+  generateBigquerySchema = () => {
+    const unsafeToSafeMappingColumnsFromColumnStore = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    if (!unsafeToSafeMappingColumnsFromColumnStore) {
+      throw new Error(`StreamId: ${this.streamId} - There is no unsafe->safe column mapping`);
+    }
+    return Object.keys(unsafeToSafeMappingColumnsFromColumnStore).reduce((acc, unsafeKey) => {
+      const renamedColName = this.renamedColumnStore.getColumnTranslation(this.streamId, unsafeKey);
+      const bigqueryTableField = this.getBigqueryTableFieldFromInternalSchemaFieldDefinition(renamedColName, this.flattenedSchema[unsafeKey]);
+      if (bigqueryTableField) {
+        acc.push(bigqueryTableField);
+      }
+      return acc;
+    }, []);
+  };
+  loadGCSFileInTable = async (tableName, file, metadata) => {
+    try {
+      const [job] = await this.bqStagingTable.load(file, metadata);
+      logger.info(`\u{1F69A} Loaded GCS file \`${file.id}\` in BigQuery table: ${tableName} with Job: \`${job.id}\``);
+      logger.debug(`Upload Stats: ${JSON.stringify(job.statistics)}`);
+      const errors2 = job.status?.errors;
+      if (errors2 && errors2.length > 0) {
+        throw errors2;
+      }
+    } catch (err) {
+      logger.error(err);
+      throw new Error(`StreamId: ${this.streamId} - Issue when loading file in BigQuery table | fileName: ${file.name}, tableName: ${tableName}
+
+            Error: ${err.message}
+            Stack: ${err.stack}`);
+    }
+  };
+};
+
+// src/targets/bigquery/main.ts
+var BigQueryTarget = class extends ITarget {
+  constructor(config, resolver) {
+    super(config, resolver);
+    this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
+  }
+  static requiredConfigKeys = [
+    "schema",
+    "project_id",
+    "loading_deck_gcs_bucket_name"
+  ];
+  newDBSyncInstance = (streamId, database, schema, table) => {
+    return new BigQueryDBSync(
+      this.config,
+      streamId,
+      database,
+      schema,
+      table,
+      this.renameColumnStore
+    );
+  };
+  genTableName = safeTableName;
+  safeColumnNameConverter = safeColumnName;
+  validateDateRange = validateDateRange;
+  convertNumberIntoDecimal = convertNumberIntoDecimal;
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   ALL_STREAM_ID_KEYWORD,
   API_CALLS_METRIC_NAME,
   Authenticator,
   BATCH_INTERVAL_MS,
-  Catalog,
+  BigQueryTarget,
   CounterMetric,
   DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME,
   ITarget,
   MissingFieldInSchemaError,
   MissingSchemaError,
-  RESTStreamV2,
+  RESTStream,
   ROWS_SYNCED_METRIC_NAME,
   RecordMessage,
   RenameColumnStore,
@@ -2839,13 +3162,12 @@ var ITarget = class _ITarget {
   SchemaValidationError,
   StateMessage,
   StateService,
+  Stream,
   StreamMetadata,
-  StreamV2,
   StreamWarehouseSyncService,
   Tap,
   _greaterThan,
   addWhalyFields,
-  checkRequiredConfigKeys,
   createTemporaryFileStream,
   extractStateForStream,
   finalizeStateProgressMarkers,
@@ -2863,8 +3185,8 @@ var ITarget = class _ITarget {
   haltAndCatchFire,
   incrementStreamState,
   loadJson,
+  loadResolver,
   loadSchema,
-  loadShoreConfig,
   logger,
   postFormDataApiCall,
   postJSONApiCall,
@@ -2872,7 +3194,6 @@ var ITarget = class _ITarget {
   printMemoryFootprint,
   removeParasiteProperties,
   safePath,
-  writeCatalogFile,
   writeStateFile
 });
 //# sourceMappingURL=index.js.map