@whaly/connector-sdk 0.1.1 → 0.3.0

package/dist/index.mjs

@@ -2,180 +2,10 @@
 import { readFileSync } from "fs";
 
 // src/sdk/service/logger.ts
-import * as winston2 from "winston";
-
-// src/sdk/models/resolver.ts
-var Resolver = class {
-  constructor() {
-  }
-};
-
-// src/resolvers/local-file/services/files.ts
-import fs from "fs";
-var writeFile = (path, content) => {
-  return Promise.resolve(fs.writeFileSync(path, content));
-};
-var readAndParseJSONFile = (path) => {
-  if (!fs.existsSync(path)) {
-    throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
-  }
-  const raw = fs.readFileSync(path);
-  const parsed = JSON.parse(raw.toString());
-  return parsed;
-};
-
-// src/resolvers/local-file/hook/schemaHook.ts
-import fsExtra from "fs-extra";
-var LocalSchemaHook = class {
-  async writeSchema(input) {
-    await fsExtra.appendFile("./schema.tmp", JSON.stringify(input) + "\n");
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-import winston from "winston";
-
-// src/sdk/service/metric.ts
-var metricStore = {};
-var ALL_STREAM_ID_KEYWORD = "all";
-var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
-var API_CALLS_METRIC_NAME = "api_calls_count";
-var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
-var getCounterMetrics = (name, streamIds) => {
-  return streamIds.map((streamId) => {
-    return getCounterMetric(name, streamId);
-  });
-};
-var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
-  if (!metricStore[streamId]) {
-    metricStore[streamId] = {};
-  }
-  const streamMetricStore = metricStore[streamId];
-  if (!streamMetricStore[name]) {
-    streamMetricStore[name] = new CounterMetric(name, streamId);
-  }
-  return streamMetricStore[name];
-};
-var getAllMetrics = () => {
-  return Object.keys(metricStore).flatMap((streamId) => {
-    const streamMetricStore = metricStore[streamId];
-    if (!streamMetricStore) {
-      return [];
-    }
-    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
-  });
-};
-var CounterMetric = class {
-  value = 0;
-  name;
-  streamId;
-  constructor(name, streamId) {
-    this.name = name;
-    this.streamId = streamId;
-  }
-  increment(inc = 1) {
-    this.value = this.value + inc;
-  }
-  getStreamId() {
-    return this.streamId;
-  }
-  getName() {
-    return this.name;
-  }
-  getValue() {
-    return this.value;
-  }
-};
-
-// src/resolvers/local-file/resolver.ts
-var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
-var statePath = "./state.json";
-var metricsPath = "./metrics.json";
-var LocalFilesResolver = class extends Resolver {
-  checkIfCanSync() {
-    return Promise.resolve(true);
-  }
-  markSyncStarted() {
-    return Promise.resolve();
-  }
-  onError(errorType, errorText, errorDebugText) {
-    const path = `./error.json`;
-    return writeFile(path, JSON.stringify({
-      errorType,
-      errorText,
-      errorDebugText
-    }));
-  }
-  getSchemaHooks() {
-    const wlyHook = new LocalSchemaHook();
-    return [wlyHook];
-  }
-  writeSchema() {
-    return Promise.resolve();
-  }
-  constructor() {
-    super();
-  }
-  getState() {
-    logger.info(`\u{1F4C1} Using the LOCAL resolver`);
-    const state = readAndParseJSONFile(statePath);
-    return Promise.resolve({
-      state
-    });
-  }
-  writeCatalog(catalog) {
-    return Promise.resolve();
-  }
-  writeState(state) {
-    return writeFile(statePath, state);
-  }
-  markSyncFailed() {
-    logger.info(`\u{1F622} Sync has failed.`);
-    return Promise.resolve();
-  }
-  markSyncComplete() {
-    logger.info(`\u{1F389} Sync is complete.`);
-    return Promise.resolve();
-  }
-  markDiscoveryComplete() {
-    logger.info(`\u{1F389} Discovery is complete.`);
-    return Promise.resolve();
-  }
-  async flushMetrics() {
-    const allMetrics = getAllMetrics();
-    const metricsOuput = allMetrics.map((metric) => {
-      return {
-        streamId: metric.getStreamId(),
-        name: metric.getName(),
-        value: metric.getValue()
-      };
-    }).map((obj) => JSON.stringify(obj)).join(`
-`);
-    return writeFile(metricsPath, metricsOuput);
-  }
-  updateSourceValue(optionKey, optionValue) {
-    logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
-    return Promise.resolve();
-  }
-  getLogFormat() {
-    return combine(
-      timestamp(),
-      errors({ stack: true }),
-      splat(),
-      prettyPrint()
-    );
-  }
-};
-
-// src/sdk/service/resolver.ts
-var loadResolver = () => {
-  return new LocalFilesResolver();
-};
-
-// src/sdk/service/logger.ts
+import * as winston from "winston";
 var BATCH_INTERVAL_MS = 100;
 var transports2 = {
-  console: new winston2.transports.Console({
+  console: new winston.transports.Console({
     stderrLevels: [],
     level: "info"
   })
@@ -185,8 +15,14 @@ var getTransports = () => {
     transports2.console
   ];
 };
-var winstonLogger = winston2.createLogger({
-  format: loadResolver().getLogFormat(),
+var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
+var winstonLogger = winston.createLogger({
+  format: combine(
+    timestamp(),
+    errors({ stack: true }),
+    splat(),
+    prettyPrint()
+  ),
   transports: getTransports()
 });
 var isWinstonOpen = true;
@@ -215,10 +51,6 @@ var logger = {
 };
 
 // src/sdk/utils.ts
-var writeStateFile = (resolver, state) => {
-  logger.info(`\u{1F4DD} Writing the state file.`);
-  return resolver.writeState(state);
-};
 function readFile(fileName, filePath) {
   try {
     return readFileSync(filePath);
@@ -248,7 +80,7 @@ var loadSchema = (streamId) => {
 // src/sdk/service/network.ts
 import axios from "axios";
 import { URLSearchParams } from "url";
-import * as fs2 from "fs";
+import * as fs from "fs";
 import * as qs from "qs";
 import axiosRetry, { isNetworkOrIdempotentRequestError } from "axios-retry";
 var shouldRetry = (error) => {
@@ -361,7 +193,7 @@ var getJSONApiCall = async (endpoint, config, privateLogging) => {
 };
 var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
   const instance = getAxiosInstance(config.retryCount);
-  const writer = fs2.createWriteStream(output);
+  const writer = fs.createWriteStream(output);
   const paramsSerializer = (params) => {
     return qs.stringify(params, { arrayFormat: "repeat" });
   };
@@ -455,6 +287,58 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
   }
 };
 
+// src/sdk/service/metric.ts
+var metricStore = {};
+var ALL_STREAM_ID_KEYWORD = "all";
+var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
+var API_CALLS_METRIC_NAME = "api_calls_count";
+var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
+var getCounterMetrics = (name, streamIds) => {
+  return streamIds.map((streamId) => {
+    return getCounterMetric(name, streamId);
+  });
+};
+var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
+  if (!metricStore[streamId]) {
+    metricStore[streamId] = {};
+  }
+  const streamMetricStore = metricStore[streamId];
+  if (!streamMetricStore[name]) {
+    streamMetricStore[name] = new CounterMetric(name, streamId);
+  }
+  return streamMetricStore[name];
+};
+var getAllMetrics = () => {
+  return Object.keys(metricStore).flatMap((streamId) => {
+    const streamMetricStore = metricStore[streamId];
+    if (!streamMetricStore) {
+      return [];
+    }
+    return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
+  });
+};
+var CounterMetric = class {
+  value = 0;
+  name;
+  streamId;
+  constructor(name, streamId) {
+    this.name = name;
+    this.streamId = streamId;
+  }
+  increment(inc = 1) {
+    this.value = this.value + inc;
+  }
+  getStreamId() {
+    return this.streamId;
+  }
+  getName() {
+    return this.name;
+  }
+  getValue() {
+    return this.value;
+  }
+};
+
 // src/sdk/service/memory.ts
 var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
 var printMemoryFootprint = (prefix) => {
@@ -470,9 +354,6 @@ var printMemoryFootprint = (prefix) => {
 
 // src/sdk/service/exit.ts
 async function gracefulExit(exitCode) {
-  if (exitCode !== 0) {
-    await loadResolver().markSyncFailed();
-  }
   const loggerFinish = new Promise((resolve, reject) => {
     logger.closeWinston(resolve);
   });
@@ -492,34 +373,16 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
     errorText,
     errorDebugText
   );
-  const resolver = loadResolver();
-  await resolver.onError(errorType, errorText, errorDebugText);
   await gracefulExit(0);
 };
 
-// src/sdk/service/relationship.ts
-var findRelatedRelationships = (streamId, allRels) => {
-  return allRels.reduce((acc, rel) => {
-    const base = {
-      type: rel.type,
-      from: rel.from,
-      to: rel.to
-    };
-    if (rel.left === streamId) {
-      acc.right.push({
-        streamId: rel.right,
-        ...base
-      });
-    }
-    if (rel.right === streamId) {
-      acc.left.push({
-        streamId: rel.left,
-        ...base
-      });
-    }
-    return acc;
-  }, { left: [], right: [] });
-};
+// src/sdk/models/replication.ts
+var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
+  ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
+  ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
+  ReplicationMethod2["APPEND"] = "APPEND";
+  return ReplicationMethod2;
+})(ReplicationMethod || {});
 
 // src/sdk/models/messages.ts
 var RecordMessage = class {
@@ -579,7 +442,6 @@ var SchemaMessage = class {
   displayLabel;
   description;
   propertiesMetadata;
-  relationships;
   constructor(opts) {
     const {
       stream,
@@ -588,7 +450,6 @@ var SchemaMessage = class {
       bookmarkProperties,
       displayLabel,
       description,
-      relationships,
       propertiesMetadata
     } = opts;
     this.stream = stream;
@@ -597,7 +458,6 @@ var SchemaMessage = class {
     this.bookmarkProperties = bookmarkProperties;
     this.displayLabel = displayLabel;
     this.description = description;
-    this.relationships = relationships;
     this.propertiesMetadata = propertiesMetadata;
   }
   toString() {
@@ -607,8 +467,7 @@ var SchemaMessage = class {
       schema: this.schema,
       key_properties: this.keyProperties,
       label: this.displayLabel,
-      description: this.description,
-      relationships: this.relationships
+      description: this.description
     };
     if (this.bookmarkProperties) {
       result["bookmark_properties"] = this.bookmarkProperties;
@@ -617,148 +476,28 @@ var SchemaMessage = class {
   }
 };
 
-// src/sdk/models/metadata.ts
-var MetadataMap = class extends Map {
-  areEquals(array1, array2) {
-    return array1.length === array2.length && array1.every(function(value, index) {
-      return value === array2[index];
-    });
-  }
-  findExistingEntry(key) {
-    return Array.from(super.entries()).find((entry) => {
-      if (this.areEquals(entry[0], key)) {
-        return true;
-      }
-    });
-  }
-  get(key) {
-    const existingEntry = this.findExistingEntry(key);
-    return existingEntry?.[1];
-  }
-  set(key, value) {
-    const existingKey = this.findExistingEntry(key);
-    if (existingKey) {
-      return super.set(existingKey?.[0], value);
-    } else {
-      return super.set(key, value);
-    }
-  }
-};
-var StreamMetadata = class {
-  metadataByBreadcrumb;
-  constructor() {
-    this.metadataByBreadcrumb = new MetadataMap();
-  }
-  fromCatalogStreamMetadata(metadataArr) {
-    metadataArr.forEach((metadata) => {
-      this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
-    });
-    return this;
-  }
-  toString() {
-    return this.metadataByBreadcrumb;
-  }
-  get(breadcrumb) {
-    return this.metadataByBreadcrumb.get(breadcrumb);
-  }
-  write(breadcrumb, key, value) {
-    if (value === void 0) {
-      throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
-            
-            Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
-    }
-    if (!this.metadataByBreadcrumb.get(breadcrumb)) {
-      this.metadataByBreadcrumb.set(breadcrumb, {});
-    }
-    const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
-    const newValue = {};
-    newValue[key] = value;
-    this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
-    return this;
-  }
-  getRootBreadcrumbMetadata() {
-    const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
-    if (!rootBreadcrumbMetadata) {
-      throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level. 
-            
-            Is your catalog properly generated?`);
-    }
-    return rootBreadcrumbMetadata;
-  }
-  isStreamSelected() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (rootBreadcrumbMetadata.selected === false) {
-      return false;
-    }
-    if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
-      return true;
-    }
-    return false;
-  }
-  getKeyProperties() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["table-key-properties"]) {
-      return [];
-    }
-    return rootBreadcrumbMetadata["table-key-properties"];
-  }
-  getReplicationKey() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-key"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-key"];
-  }
-  getReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["replication-method"];
-  }
-  getForcedReplicationMethod() {
-    const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
-    if (!rootBreadcrumbMetadata["forced-replication-method"]) {
-      return void 0;
-    }
-    return rootBreadcrumbMetadata["forced-replication-method"];
-  }
-  toList() {
-    return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
-      const metadata = this.metadataByBreadcrumb.get(breadcrumb);
-      if (!metadata) {
-        throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
-      }
-      return {
-        breadcrumb,
-        metadata
-      };
-    });
-  }
-};
-
 // src/sdk/models/state.ts
-import moment from "moment";
+import dayjs from "dayjs";
 
 // src/sdk/constants/date.ts
 var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
 
 // src/sdk/models/state.ts
-import { format } from "util";
+import { format as format2 } from "util";
 var PROGRESS_MARKERS = "progressMarkers";
 var PROGRESS_MARKER_NOTE = "Note";
 var SIGNPOST_MARKER = "replicationKeySignpost";
 var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
   let newRkValue = latestRecord[replicationKey];
   if (!newRkValue) {
-    throw new Error(format(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
+    throw new Error(format2(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
   }
-  let newRkValueMoment = moment(newRkValue);
+  let newRkValueMoment = dayjs(newRkValue);
   let prevRkValue;
   let prevRkValueMoment;
   if (isSorted) {
     prevRkValue = state["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue);
+    prevRkValueMoment = dayjs(prevRkValue);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       throw new Error(
         `Unsorted data detected in stream. Latest value '${newRkValue}' is
@@ -773,24 +512,24 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
       state[PROGRESS_MARKERS] = marker;
     }
     prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
-    prevRkValueMoment = moment(prevRkValue || state["replicationKeyValue"]);
+    prevRkValueMoment = dayjs(prevRkValue || state["replicationKeyValue"]);
     if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
       newRkValueMoment = prevRkValueMoment;
       newRkValue = prevRkValue;
     }
   }
   const signpostMarker = state[SIGNPOST_MARKER];
-  const replicationKeySignpostMoment = moment(signpostMarker);
+  const replicationKeySignpostMoment = dayjs(signpostMarker);
   if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
     newRkValueMoment = replicationKeySignpostMoment;
     newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
   }
   if (isSorted === false) {
     state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
-    state[PROGRESS_MARKERS]["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state[PROGRESS_MARKERS]["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   } else {
     state["replicationKey"] = replicationKey;
-    state["replicationKeyValue"] = moment(newRkValueMoment).format(defaultDateTimeFormat);
+    state["replicationKeyValue"] = dayjs(newRkValueMoment).format(defaultDateTimeFormat);
   }
   return state;
 };
@@ -812,7 +551,7 @@ var finalizeStateProgressMarkers = (state) => {
   return state;
 };
 var _greaterThan = (aValue, bValue) => {
-  return moment(aValue).isAfter(moment(bValue));
+  return dayjs(aValue).isAfter(dayjs(bValue));
 };
 var extractStateForStream = (state, streamId) => {
   if (!state.bookmarks) {
@@ -835,26 +574,16 @@ var StateService = class {
     }
     return this.instance;
   }
-  // To be deprecated
-  setBookmark(streamId, ts) {
-    this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
-  }
-  setBookmarkV2(streamId, streamState) {
+  setBookmark(streamId, streamState) {
     this.bookmarks[streamId] = streamState;
   }
-  setBookmarkSignpostV2(streamId, signpostValue) {
+  setBookmarkSignpost(streamId, signpostValue) {
     if (!this.bookmarks[streamId]) {
       this.bookmarks[streamId] = {};
     }
     this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
   }
   getBookmark(streamId) {
-    if (!this.bookmarks[streamId]) {
-      return moment(0);
-    }
-    return moment(this.bookmarks[streamId]);
-  }
-  getBookmarkV2(streamId) {
     if (this.bookmarks[streamId]) {
       return this.bookmarks[streamId];
     } else {
@@ -894,7 +623,7 @@ import axios2 from "axios";
 import * as qs2 from "qs";
 
 // src/sdk/models/tap/stream.ts
-import moment2 from "moment";
+import dayjs2 from "dayjs";
 
 // src/sdk/helpers/typing.ts
 var isDatetimeType = (typeDef) => {
@@ -917,10 +646,11 @@ var isDatetimeType = (typeDef) => {
 };
 
 // src/sdk/models/tap/stream.ts
-import cloneDeep from "lodash/cloneDeep.js";
+import cloneDeep2 from "lodash/cloneDeep.js";
 import Bluebird2 from "bluebird";
 
 // src/sdk/models/tap/tap.ts
+import cloneDeep from "lodash/cloneDeep.js";
 import Bluebird from "bluebird";
 var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
 var Tap = class {
@@ -928,12 +658,21 @@ var Tap = class {
   config;
   streams;
   concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
-  constructor(target, config) {
+  stateProvider;
+  tapState;
+  constructor(target, config, stateProvider) {
     this.streams = [];
     this.target = target;
     this.config = config;
+    this.stateProvider = stateProvider;
+    this.tapState = { bookmarks: {} };
   }
   sync = async (options) => {
+    const initialState = await this.stateProvider.getState();
+    if (initialState.state && initialState.state.bookmarks) {
+      this.tapState = cloneDeep(initialState.state);
+    }
+    await this.init();
     logger.info(`\u{1F680} Start syncing`);
     const state = StateService.getInstance().get();
     logger.info(`\u{1F4CD} Received state: %j`, state);
@@ -964,15 +703,13 @@ var Tap = class {
 };
 
 // src/sdk/models/tap/stream.ts
-import { format as format2 } from "util";
+import { format as format3 } from "util";
 var Stream = class {
   // Runtime values, can't be overriden
   config;
   tapState;
   target;
-  // Compile time values, can be overriden
-  // Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
-  forcedReplicationMethod = void 0;
+  replicationMethod = void 0;
   selectedByDefault = true;
   STATE_MSG_FREQUENCY = 10;
   streamId = "default";
@@ -980,10 +717,9 @@ var Stream = class {
   displayLabel = void 0;
   description = void 0;
   primaryKey = [];
-  forcedReplicationKey = void 0;
+  replicationKey = void 0;
   // When set, use the state from another stream for this stream
   useStateFromStreamId = void 0;
-  relationships = [];
   children = [];
   // If true, it means that the records in this stream are sorted by replicationKey.
   // If false, then the records are coming in an unsorted manner.
@@ -995,9 +731,9 @@ var Stream = class {
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
-  constructor(config, state, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
+  constructor(config, tapState, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
     this.config = config;
-    this.tapState = cloneDeep(state);
+    this.tapState = cloneDeep2(tapState);
     this.target = target;
     this.childConcurrency = childConcurrency;
     this.rowsSyncedMetricsConf = {
@@ -1029,7 +765,7 @@ var Stream = class {
       return this.flush();
     } catch (err) {
       if (err instanceof Error) {
-        throw new Error(format2(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
+        throw new Error(format3(`got an error while syncing streamId=%s, err: %s`, this.streamId, err.message));
       }
       throw err;
     }
@@ -1070,13 +806,11 @@ var Stream = class {
     if (!this.isSilent) {
       const schema = await this.getSchema();
       if (schema !== void 0) {
-        const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
         const replicationConfig = this.configuredReplicationConfig();
         const message = new SchemaMessage({
           keyProperties: this.primaryKey,
           stream: this.streamId,
           schema: schema.jsonSchema,
-          relationships: relatedRels,
           ...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
           ...this.displayLabel ? { displayLabel: this.displayLabel } : {},
           ...this.description ? { description: this.description } : {},
@@ -1096,7 +830,7 @@ var Stream = class {
     const signpostMoment = await this.getReplicationKeySignpost();
     const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
     if (signpostValue) {
-      StateService.getInstance().setBookmarkSignpostV2(this.streamId, signpostValue);
+      StateService.getInstance().setBookmarkSignpost(this.streamId, signpostValue);
     }
   };
   /**
@@ -1112,7 +846,7 @@ var Stream = class {
         if (this.useStateFromStreamId) {
           return;
         }
-        if (this.forcedReplicationMethod && !this.forcedReplicationKey) {
+        if (this.replicationMethod && !this.replicationKey) {
           return;
         }
         if (!replicationConfig.replicationKey) {
@@ -1122,14 +856,14 @@ var Stream = class {
           );
         }
         const stateServiceInst = StateService.getInstance();
-        const state = stateServiceInst.getBookmarkV2(this.streamId);
+        const state = stateServiceInst.getBookmark(this.streamId);
         const newState = incrementStreamState(
           state,
           replicationConfig.replicationKey,
           latestRecord,
           this.isSorted
         );
-        StateService.getInstance().setBookmarkV2(this.streamId, newState);
+        StateService.getInstance().setBookmark(this.streamId, newState);
       }
     }
   };
@@ -1169,19 +903,19 @@ var Stream = class {
         this.replicationKey: ${replicationConfig.replicationKey}`);
     if (state.replicationKeyValue) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
-      return moment2(state.replicationKeyValue);
+      return dayjs2(state.replicationKeyValue);
     } else if (startDate) {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
-      return moment2(startDate);
+      return dayjs2(startDate);
     } else {
       logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
-      return moment2(0);
+      return dayjs2(0);
     }
   }
   /**
    * Return the max allowable bookmark value for this stream's replication key.
    * 
-   * For timestamp-based replication keys, this defaults to `moment()`. For
+   * For timestamp-based replication keys, this defaults to `dayjs()`. For
    * non-timestamp replication keys, default to `undefined`.
    * 
    * Override this value to prevent bookmarks from being advanced in cases where we
@@ -1189,7 +923,7 @@ var Stream = class {
    */
   getReplicationKeySignpost = async () => {
     if (await this.isTimestampReplicationKey()) {
-      return moment2();
+      return dayjs2();
     }
     return void 0;
   };
@@ -1197,22 +931,22 @@ var Stream = class {
    * Return the default replication method for the stream that will be used to write the catalog.
    */
   defaultReplicationConfig = () => {
-    if (this.forcedReplicationMethod) {
+    if (this.replicationMethod) {
       return {
-        replicationMethod: this.forcedReplicationMethod,
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: this.replicationMethod,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
-    if (this.forcedReplicationKey || this.useStateFromStreamId) {
+    if (this.replicationKey || this.useStateFromStreamId) {
       return {
-        replicationMethod: "INCREMENTAL",
-        replicationKey: this.forcedReplicationKey,
+        replicationMethod: "INCREMENTAL" /* INCREMENTAL */,
+        replicationKey: this.replicationKey,
         isForced: true
       };
     }
     return {
-      replicationMethod: "FULL_TABLE",
+      replicationMethod: "FULL_TABLE" /* FULL_TABLE */,
       replicationKey: void 0,
       isForced: false
     };
@@ -1273,9 +1007,9 @@ var Stream = class {
       rowsSent += 1;
     }
     const stateServiceInst = StateService.getInstance();
-    const state = stateServiceInst.getBookmarkV2(this.streamId);
+    const state = stateServiceInst.getBookmark(this.streamId);
     const newState = finalizeStateProgressMarkers(state);
-    stateServiceInst.setBookmarkV2(this.streamId, newState);
+    stateServiceInst.setBookmark(this.streamId, newState);
     if (!parent) {
       logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
     } else {
@@ -1381,7 +1115,7 @@ function getTruncatedParamsForLog(params) {
 }
 
 // src/sdk/models/tap/restStream.ts
-import cloneDeep2 from "lodash/cloneDeep.js";
+import cloneDeep3 from "lodash/cloneDeep.js";
 import isEqual from "lodash/isEqual.js";
 var RESTStream = class extends Stream {
   axiosInstance;
@@ -1402,13 +1136,24 @@ var RESTStream = class extends Stream {
    */
   _prepareRequest = async (nextPageToken, parent) => {
     const method = this.httpMethod;
-    const url = this.getUrl(parent);
+    const url = this.getNextUrl(nextPageToken, parent);
     if (!url) {
       throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
     }
     const authenticator = this.authenticator;
+    let finalUrl = url;
+    let urlQueryParams = {};
+    const queryIndex = url.indexOf("?");
+    if (queryIndex !== -1) {
+      finalUrl = url.substring(0, queryIndex);
+      const rawQueryString = url.substring(queryIndex + 1);
+      urlQueryParams = qs2.parse(rawQueryString);
+    }
     const params = {
       ...this.getNextUrlParams(nextPageToken, parent),
+      // Query params embedded in URL should override computed pagination params
+      // but should not override auth params which are appended last
+      ...urlQueryParams,
       ...await authenticator?.getAuthQS(this.config)
     };
     const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
@@ -1427,7 +1172,7 @@ var RESTStream = class extends Stream {
       return qs2.stringify(params2, { arrayFormat: "repeat" });
     };
     const request = {
-      url,
+      url: finalUrl,
       method,
       data: requestBody,
       params,
@@ -1500,7 +1245,7 @@ var RESTStream = class extends Stream {
       for await (const row of rows) {
         yield row;
       }
-      const previousToken = cloneDeep2(nextPageToken);
+      const previousToken = cloneDeep3(nextPageToken);
       nextPageToken = this.getNextPageToken(resp, previousToken);
       if (nextPageToken && isEqual(nextPageToken, previousToken)) {
         throw new Error(
@@ -1579,7 +1324,7 @@ var RESTStream = class extends Stream {
    * TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
    * @returns 
    */
-  getUrl(parent) {
+  getNextUrl(previousToken, parent) {
     const urlPattern = [this.baseUrl, this.path].join("");
     return urlPattern;
   }
@@ -1760,6 +1505,8 @@ var StreamWarehouseSyncService = class {
       mergeQueries = this.getReplaceQueries();
     } else if (replicationMethod === "INCREMENTAL") {
       mergeQueries = this.getMergeQueries();
+    } else if (replicationMethod === "APPEND") {
+      mergeQueries = this.getAppendQueries();
     } else if (replicationMethod === "LOG_BASED") {
       mergeQueries = this.getMergeQueries();
     }
@@ -1931,25 +1678,25 @@ var flattenSchema = (streamId, schema) => {
 
 // src/sdk/models/target/target.ts
 import { Validator } from "jsonschema";
-import moment4 from "moment";
+import dayjs4 from "dayjs";
 
 // src/sdk/models/target/streamDbState.ts
-import moment3 from "moment";
+import dayjs3 from "dayjs";
 
 // src/sdk/models/target/temporaryFile.ts
-import fs3 from "fs-extra";
+import fs2 from "fs-extra";
 import uniqueId from "lodash/uniqueId.js";
 var createTemporaryFileStream = (streamId) => {
-  fs3.ensureDirSync("./tmp");
-  const path = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
-  const writeStream = fs3.createWriteStream(path, { flags: "w" });
+  fs2.ensureDirSync("./tmp");
+  const path2 = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
+  const writeStream = fs2.createWriteStream(path2, { flags: "w" });
   return {
     stream: writeStream,
-    path
+    path: path2
   };
 };
-function safePath(path) {
-  let formattedPath = path;
+function safePath(path2) {
+  let formattedPath = path2;
   formattedPath = formattedPath.replace(/^The\s+/gi, "");
   formattedPath = formattedPath.replace(/\s+(?:\(|\[)?Dis(?:c|k) (?:One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\d)+(?:\)|\])?/gi, "");
   formattedPath = formattedPath.replace(/\s+/gi, "_");
@@ -1990,7 +1737,7 @@ function replaceSymbols(str) {
 }
 
 // src/sdk/models/target/streamDbState.ts
-import * as fs4 from "fs";
+import * as fs3 from "fs";
 var StreamState = class {
   streamId;
   schema;
@@ -2015,7 +1762,7 @@ var StreamState = class {
     this.fileToLoad = createTemporaryFileStream(streamId);
     this.syncedRowCount = 0;
     this.keyProperties = [];
-    this.batchDate = moment3();
+    this.batchDate = dayjs3();
     this.replicationMethod = replicationMethod;
     this._hasBeenLoadedYetDuringThisSync = false;
   }
@@ -2040,7 +1787,7 @@ var StreamState = class {
   resetFileToLoad() {
     this.fileToLoad.stream.close();
     const oldPath = this.fileToLoad.path;
-    fs4.unlinkSync(oldPath);
+    fs3.unlinkSync(oldPath);
     this.fileToLoad = createTemporaryFileStream(this.streamId);
     this.batchedRowCount = 0;
   }
@@ -2071,8 +1818,8 @@ var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
   schemaHooks;
-  resolver;
   syncTime;
+  stateProvider;
   // Latest state message received from the Tap
   // Not yet flushed as we didn't upload the stream data since receiving it 
   batchedState;
@@ -2083,15 +1830,15 @@ var ITarget = class _ITarget {
   streams;
   // JSON Schema validator
   validator;
-  constructor(config, resolver) {
+  constructor(config, stateProvider) {
     this.config = config;
-    this.resolver = resolver;
-    this.schemaHooks = resolver.getSchemaHooks();
+    this.stateProvider = stateProvider;
+    this.schemaHooks = [];
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
     this.validator = new Validator();
-    this.syncTime = moment4();
+    this.syncTime = dayjs4();
     this.renameColumnStore = new RenameColumnStore();
   }
   ///////////////////////////////////////////
@@ -2200,7 +1947,7 @@ var ITarget = class _ITarget {
     this.streams[streamId] = new StreamState(
       streamId,
       dbSyncInstance,
-      replicationMethod || "FULL_TABLE"
+      replicationMethod || "FULL_TABLE" /* FULL_TABLE */
     );
   }
   replicationMethod = (message) => {
@@ -2258,7 +2005,8 @@ var ITarget = class _ITarget {
         }
       });
       streamState.setSchema(schema);
-      if (message.keyProperties.length === 0) {
+      const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
+      if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
             Received SCHEMA message: ${JSON.stringify(message)}`);
       }
@@ -2277,39 +2025,7 @@ var ITarget = class _ITarget {
             return translation;
           }
           return k;
-        }),
-        relationships: {
-          left: message.relationships.left.flatMap((l) => {
-            if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
-              const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
-              const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
-              if (from && to) {
-                return [{
-                  type: l.type,
-                  streamId: l.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          }),
-          right: message.relationships.right.flatMap((r) => {
-            if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
-              const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
-              const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
-              if (from && to) {
-                return [{
-                  type: r.type,
-                  streamId: r.streamId,
-                  from,
-                  to
-                }];
-              }
-            }
-            return [];
-          })
-        }
+        })
       };
       await Bluebird3.map(this.schemaHooks, async (hook) => {
         const input = {
@@ -2342,9 +2058,6 @@ var ITarget = class _ITarget {
     try {
       logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
       await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
-      const configResolver = loadResolver();
-      await configResolver.markSyncComplete();
-      await configResolver.flushMetrics();
       return Promise.resolve();
     } catch (err) {
       logger.error(`Error while handling complete event.`);
@@ -2357,7 +2070,8 @@ var ITarget = class _ITarget {
       return false;
     }
     const batchedRecords = stream.getBatchedRowCount();
-    if (stream.getReplicationMethod() === "INCREMENTAL" && batchedRecords > 1e6) {
+    const method = stream.getReplicationMethod();
+    if ((method === "INCREMENTAL" || method === "APPEND") && batchedRecords > 1e6) {
       logger.info(`\u{1F9CA} Stream=${streamId} has reached the maximum amount of batched records. We'll trigger a flush of ALL streams in the Warehouse.`);
       return true;
     } else {
@@ -2387,8 +2101,7 @@ var ITarget = class _ITarget {
     }
   };
   emitState = (state) => {
-    const configResolver = loadResolver();
-    return writeStateFile(configResolver, JSON.stringify(state));
+    return this.stateProvider.writeState(JSON.stringify(state));
   };
   uploadStreamsToWarehouse = async (streamsToUpload) => {
     try {
@@ -2427,24 +2140,1089 @@ var ITarget = class _ITarget {
   };
 };
 
-// src/targets/bigquery/helpers.ts
-var safeColumnName = (key) => {
-  let returnString = key;
-  const shouldNotStartWith = ["_TABLE_", "_FILE_", "_PARTITION"];
-  shouldNotStartWith.forEach((snm) => {
-    if (returnString.startsWith(snm)) {
-      returnString = returnString.replace(snm, "");
+// src/sdk/file-processing/types.ts
+var FileFormat = /* @__PURE__ */ ((FileFormat2) => {
+  FileFormat2["CSV"] = "CSV";
+  FileFormat2["EXCEL"] = "EXCEL";
+  return FileFormat2;
+})(FileFormat || {});
+
+// src/sdk/file-processing/schema-utils.ts
+function fieldTypeToJsonSchema(fieldType) {
+  switch (fieldType) {
+    case "STRING":
+      return { type: ["null", "string"] };
+    case "FLOAT":
+      return { type: ["null", "number"] };
+    case "TIMESTAMP":
+      return { type: ["null", "string"], format: "date-time" };
+    default:
+      return { type: ["null", "string"] };
+  }
+}
+function excelFieldsToJsonSchema(fields) {
+  const properties = {};
+  for (const [key, field] of Object.entries(fields)) {
+    properties[key] = fieldTypeToJsonSchema(field.type);
+  }
+  return { type: "object", properties };
+}
+function csvFieldsToJsonSchema(config) {
+  const properties = {};
+  if (Array.isArray(config)) {
+    for (const field of config) {
+      properties[field.key] = fieldTypeToJsonSchema(field.type);
+    }
+  } else {
+    for (const [, field] of Object.entries(config)) {
+      properties[field.key] = fieldTypeToJsonSchema(field.type);
+    }
+  }
+  return { type: "object", properties };
+}
+function extractPrimaryKeysFromCsvConfig(config) {
+  if (Array.isArray(config)) {
+    return config.map((f) => f.key);
+  }
+  return Object.values(config).map((f) => f.key);
+}
+function extractPrimaryKeysFromExcelFields(fields) {
+  return Object.entries(fields).filter(([, field]) => {
+    return field.primaryKey === true || field.primaryKey === true;
+  }).map(([key]) => key);
+}
+
+// src/sdk/file-processing/excel/reader.ts
+import XLSX from "xlsx";
+import path from "path";
+var excelColumnToIndex = (columnLetter) => {
+  const letters = columnLetter.toUpperCase();
+  let index = 0;
+  for (let i = 0; i < letters.length; i++) {
+    const charCode = letters.charCodeAt(i);
+    if (charCode < 65 || charCode > 90) {
+      throw new Error(`Invalid column letter: ${columnLetter}`);
+    }
+    index = index * 26 + (charCode - 64);
+  }
+  return index - 1;
+};
+var indexToExcelColumn = (columnIndex) => {
+  if (columnIndex < 0) {
+    throw new Error(`Invalid column index: ${columnIndex}. Must be >= 0`);
+  }
+  let result = "";
+  let index = columnIndex;
+  while (true) {
+    result = String.fromCharCode(65 + index % 26) + result;
+    index = Math.floor(index / 26);
+    if (index === 0) break;
+    index--;
+  }
+  return result;
+};
+var excelRowToIndex = (rowNumber) => {
+  const rowNum = typeof rowNumber === "string" ? parseInt(rowNumber, 10) : rowNumber;
+  if (isNaN(rowNum) || rowNum < 1) {
+    throw new Error(`Invalid row number: ${rowNumber}. Must be >= 1`);
+  }
+  return rowNum - 1;
+};
+var indexToExcelRow = (index) => {
+  if (index < 0) {
+    throw new Error(`Invalid index: ${index}. Must be >= 0`);
+  }
+  return index + 1;
+};
+var parseCellReference = (cellRef) => {
+  const match = cellRef.match(/^([A-Z]+)(\d+)$/);
+  if (!match) {
+    throw new Error(`Invalid cell reference: ${cellRef}. Expected format like "A1", "Z24", "AA100"`);
+  }
+  const [, columnLetter, rowNumber] = match;
+  return {
+    columnIndex: excelColumnToIndex(columnLetter),
+    rowIndex: excelRowToIndex(rowNumber)
+  };
+};
+var createCellReference = (columnIndex, rowIndex) => {
+  return indexToExcelColumn(columnIndex) + indexToExcelRow(rowIndex);
+};
+var extractSingleSheetRows = async (fileName, conf) => {
+  console.log("Extracting single sheet rows from file: %s", fileName);
+  const workbook = XLSX.readFile(fileName, { dense: true });
+  const sheetKeys = Object.keys(workbook.Sheets);
+  const sheet = conf.sheetName ? workbook.Sheets[conf.sheetName] : workbook.Sheets[sheetKeys[0]];
+  if (!sheet) {
+    throw new Error(`Sheet ${conf.sheetName} not found in workbook`);
+  }
+  const sheetData = sheet["!data"];
+  if (!sheetData) {
+    throw new Error(`No data in ${conf.sheetName || "first"} sheet`);
+  }
+  if (sheetData.length === 0) {
+    throw new Error(`Sheet ${conf.sheetName || "first"} is empty`);
+  }
+  const variables = conf.fileNameVariablesExtractor(fileName, workbook);
+  const data = [];
+  let rowIdx = 0;
+  const minColumnIndex = Object.values(conf.columns).reduce((min, column) => {
+    return column.column ? Math.min(min, excelColumnToIndex(column.column)) : min;
+  }, Number.MAX_SAFE_INTEGER);
+  for (const row of sheetData) {
+    if (rowIdx < conf.numberOfRowsToSkip) {
+      rowIdx++;
+      continue;
+    }
+    if (!row) {
+      console.log("Empty row at index: %s, skipped processing.", rowIdx);
+      continue;
+    }
+    if (minColumnIndex !== Number.MAX_SAFE_INTEGER && !row[minColumnIndex]?.v) {
+      console.log("stopping processing because of empty value in the first data column (idx: %s)", minColumnIndex);
+      break;
+    }
+    const rowData = Object.entries(conf.columns).reduce((acc, [key, column]) => {
+      if (column.variableName) {
+        if (!variables) {
+          throw new Error(`No variables extracted from filename, cannot extract derived field ${key}`);
+        }
+        acc[key] = variables[column.variableName] || "";
+      } else {
+        const colIndex = excelColumnToIndex(column.column);
+        acc[key] = row[colIndex]?.v || "";
+      }
+      return acc;
+    }, {});
+    data.push(rowData);
+    rowIdx++;
+  }
+  console.log("Extracted %s rows from sheet %s", data.length, conf.sheetName || "first");
+  return data;
+};
+var extractExcelRows = async (localFilePath, conf) => {
+  let data;
+  if (conf.type === "single-sheet-extraction") {
+    data = await extractSingleSheetRows(localFilePath, conf);
+  } else if (conf.type === "processor") {
+    const workbook = XLSX.readFile(localFilePath, { dense: true });
+    data = await conf.processor(workbook);
+  } else {
+    throw new Error(`Unsupported configuration type: ${conf.type}`);
+  }
+  return data;
+};
+var validateExcelConfig = (config) => {
+  if (!config.extension) {
+    throw new Error("Extension is required");
+  }
+  if (!config.tableName) {
+    throw new Error("Table name is required");
+  }
+  if (!config.fileNameValidator || typeof config.fileNameValidator !== "function") {
+    throw new Error("File name validator function is required");
+  }
+  if (!config.fileNameVariablesExtractor || typeof config.fileNameVariablesExtractor !== "function") {
+    throw new Error("File name variables extractor function is required");
+  }
+  if (config.type === "single-sheet-extraction" && (!config.columns || Object.keys(config.columns).length === 0)) {
+    throw new Error("At least one column configuration is required for single-sheet-extraction");
+  }
+  if (config.type === "single-sheet-extraction") {
+    Object.entries(config.columns).forEach(([key, column]) => {
+      if (!column.type) {
+        throw new Error(`Column ${key} must have a type`);
+      }
+      if (column.column) {
+        const colLetter = column.column;
+        if (!/^[A-Z]+$/i.test(colLetter)) {
+          throw new Error(`Invalid column letter format: ${colLetter}`);
+        }
+      } else if (column.variableName) {
+      } else {
+        throw new Error(`Column ${key} must specify either 'column' or 'variableName'`);
+      }
+    });
+    if (config.numberOfRowsToSkip === void 0 || config.numberOfRowsToSkip < 0) {
+      throw new Error("Number of rows to skip must be specified and non-negative");
+    }
+  } else if (config.type === "processor") {
+    if (!config.processor || typeof config.processor !== "function") {
+      throw new Error("Processor function is required for processor type");
+    }
+  }
+};
+var findAllMatchingConfigs = (filename, configs) => {
+  const fileInfo = path.parse(filename);
+  return configs.filter(
+    (config) => config.fileNameValidator(fileInfo.base) && config.extension === fileInfo.ext.replace(".", "")
+  );
+};
+async function* createExcelGenerator(data) {
+  for (const row of data) {
+    yield row;
+  }
+}
+
+// src/sdk/file-processing/excel/config-builder.ts
+var ExcelExtractionConfigBuilder = class _ExcelExtractionConfigBuilder {
+  config = {
+    fileNameValidator: () => true,
+    fileNameVariablesExtractor: () => ({}),
+    replicationMethod: "FULL_TABLE" /* FULL_TABLE */
+  };
+  static create() {
+    return new _ExcelExtractionConfigBuilder();
+  }
+  extension(ext) {
+    this.config.extension = ext;
+    return this;
+  }
+  tableName(name) {
+    this.config.tableName = name;
+    return this;
+  }
+  fileValidator(validator) {
+    this.config.fileNameValidator = validator;
+    return this;
+  }
+  variablesExtractor(extractor) {
+    this.config.fileNameVariablesExtractor = extractor;
+    return this;
+  }
+  replicationMethod(method) {
+    this.config.replicationMethod = method;
+    return this;
+  }
+  columns(cols) {
+    if (this.config.type === "single-sheet-extraction") {
+      this.config.columns = cols;
+    } else {
+      throw new Error("Columns can only be set for single-sheet-extraction");
+    }
+    return this;
+  }
+  singleSheet(sheetName, skipRows = 0) {
+    this.config.type = "single-sheet-extraction";
+    if (sheetName !== void 0) {
+      this.config.sheetName = sheetName;
+    }
+    this.config.numberOfRowsToSkip = skipRows;
+    return this;
+  }
+  processor(processorFn) {
+    this.config.type = "processor";
+    this.config.processor = processorFn;
+    return this;
+  }
+  build() {
+    if (!this.config.type) {
+      throw new Error("Configuration type must be specified (singleSheet or processor)");
+    }
+    if (!this.config.extension) {
+      throw new Error("Extension must be set");
+    }
+    if (!this.config.tableName) {
+      throw new Error("Table name must be set");
+    }
+    if (!this.config.replicationMethod) {
+      throw new Error("Replication method must be set");
+    }
+    if (typeof this.config.fileNameValidator !== "function") {
+      throw new Error("fileNameValidator must be a function");
+    }
+    if (typeof this.config.fileNameVariablesExtractor !== "function") {
+      throw new Error("fileNameVariablesExtractor must be a function");
+    }
+    if (this.config.type === "single-sheet-extraction") {
+      const singleSheetConfig = this.config;
+      if (!singleSheetConfig.columns || Object.keys(singleSheetConfig.columns).length === 0) {
+        throw new Error("Columns must be set with at least one column for single-sheet-extraction");
+      }
+    }
+    if (this.config.type === "processor") {
+      const processorConfig = this.config;
+      if (typeof processorConfig.processor !== "function") {
+        throw new Error("Processor function must be set for processor configs");
+      }
+    }
+    return this.config;
+  }
+};
+
+// src/sdk/file-processing/csv/reader.ts
+import csv from "csv-parser";
+import { createReadStream } from "fs";
+import { format as format4 } from "util";
+import { Readable, Transform } from "stream";
+import * as iconv from "iconv-lite";
+var logPrefix = "[csvReader]";
+var createStripBomTransform = () => {
+  let isFirstChunk = true;
+  return new Transform({
+    transform(chunk2, encoding, callback) {
+      if (isFirstChunk) {
+        isFirstChunk = false;
+        if (chunk2.length >= 3 && chunk2[0] === 239 && chunk2[1] === 187 && chunk2[2] === 191) {
+          chunk2 = chunk2.slice(3);
+        }
+      }
+      callback(null, chunk2);
+    }
+  });
+};
+var getStringHexInfo = (str) => {
+  const bytes = Buffer.from(str, "utf8");
+  const hex = bytes.toString("hex").match(/.{2}/g)?.join(" ") || "";
+  return {
+    original: str,
+    length: str.length,
+    hex,
+    trimmed: str.trim()
+  };
+};
+var formatHexComparison = (expected, actual) => {
+  const expectedInfo = getStringHexInfo(expected);
+  const actualInfo = getStringHexInfo(actual);
+  return `
+Expected: "${expected}" (${expectedInfo.length} chars) [${expectedInfo.hex}]
+Actual:   "${actual}" (${actualInfo.length} chars) [${actualInfo.hex}]`;
+};
+async function* rowGeneratorFromCsv(path2, fileConfig) {
+  const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+  const csvOptions = { separator: fileConfig.separator };
+  if (isGeneratorConfigArray) {
+    csvOptions.headers = false;
+  }
+  const csvStream = csv(csvOptions);
+  const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
+  let initialReadStream;
+  let finalInputStream;
+  const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
+  if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
+    initialReadStream = createReadStream(path2);
+    finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
+  } else {
+    initialReadStream = createReadStream(path2);
+    finalInputStream = initialReadStream.pipe(createStripBomTransform());
+  }
+  finalInputStream.pipe(csvStream);
+  try {
+    for await (const row of Readable.from(csvStream)) {
+      if (!isGeneratorConfigArray) {
+        const rowData = Object.keys(fileConfig.fields).reduce((acc, key) => {
+          const keyGenerator = fileConfig.fields[key.trim()];
+          if (!keyGenerator) {
+            return acc;
+          }
+          return {
+            ...acc,
+            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key]
+          };
+        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+        yield rowData;
+      } else {
+        const rowData = Object.keys(row).reduce((acc, key, i) => {
+          const keyGenerator = fileConfig.fields[i];
+          if (!keyGenerator) {
+            return acc;
+          }
+          return {
+            ...acc,
+            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i]
+          };
+        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+        yield rowData;
+      }
+    }
+  } finally {
+    initialReadStream.destroy();
+  }
+}
+var checkCsvHeaderRow = async (path2, fileConfig) => {
+  return new Promise((resolve, reject) => {
+    const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
+    console.debug(`${logPrefix} CSV Expected columns:`, colsWithParsingConfig.map((col) => {
+      const info = getStringHexInfo(col);
+      return `"${info.original}" [${info.hex}]`;
+    }));
+    const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+    const csvOptions2 = { separator: fileConfig.separator };
+    if (isGeneratorConfigArray) {
+      csvOptions2.headers = false;
+    }
+    const csvStream = csv(csvOptions2);
+    let initialReadStream;
+    let finalInputStream;
+    const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
+    if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
+      initialReadStream = createReadStream(path2);
+      finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
+    } else {
+      initialReadStream = createReadStream(path2);
+      finalInputStream = initialReadStream.pipe(createStripBomTransform());
+    }
+    let isFirstLine = true;
+    finalInputStream.pipe(csvStream).on("data", (row) => {
+      if (isFirstLine) {
+        let headers = [];
+        if (isGeneratorConfigArray) {
+          headers = Object.values(row).filter((r) => !!r);
+        } else {
+          headers = Object.keys(row).filter((r) => !!r);
+        }
+        console.debug(`${logPrefix} CSV Detected headers:`, headers.map((h, idx) => {
+          const info = getStringHexInfo(h);
+          return `[${idx}]: "${info.original}" [${info.hex}]`;
+        }));
+        if (isGeneratorConfigArray) {
+          colsWithParsingConfig.forEach((col, i) => {
+            if (col !== headers[i]) {
+              const hexComparison = formatHexComparison(col, headers[i] || "<undefined>");
+              console.error(`${logPrefix} CSV Header Mismatch:${hexComparison}`);
+              throw new Error(format4(
+                `CSV header mismatch at position %s: expected '%s' but got '%s' in file %s`,
+                i,
+                col,
+                headers[i] || "<undefined>",
+                path2
+              ));
+            }
+          });
+        } else {
+          colsWithParsingConfig.forEach((col) => {
+            if (!headers.includes(col)) {
+              const colInfo = getStringHexInfo(col);
+              console.error(`CSV Missing column: "${col}" [${colInfo.hex}]`);
+              console.error(`Available headers:`, headers.map((h) => `"${h}" [${getStringHexInfo(h).hex}]`));
+              const trimmedMatch = headers.find((h) => h.trim() === col.trim());
+              if (trimmedMatch) {
+                console.error(`Potential trimmed match: "${trimmedMatch}" [${getStringHexInfo(trimmedMatch).hex}]`);
+              }
+              throw new Error(format4(
+                `CSV missing expected column '%s' in file: %s`,
+                col,
+                path2
+              ));
+            }
+          });
+        }
+        isFirstLine = false;
+      } else {
+        initialReadStream.destroy();
+      }
+    }).on("error", (error) => {
+      reject(new Error(`Error processing CSV file: ${error.message}`));
+    });
+    initialReadStream.on("close", () => {
+      resolve();
+    });
+  });
+};
+
+// src/sdk/file-processing/csv/writer.ts
+import { createObjectCsvWriter } from "csv-writer";
+import { format as format5 } from "util";
+var logPrefix2 = "[csvWriter]";
+var writeDataToCsv = async (fileName, data) => {
+  try {
+    const firstRow = data[0];
+    if (!firstRow) {
+      console.log(`${logPrefix2} No data to write to CSV file=%s`, fileName);
+      return;
+    }
+    const csvWriter = createObjectCsvWriter({
+      path: fileName,
+      header: Object.keys(firstRow).map((col) => {
+        return { id: col, title: col };
+      })
+    });
+    await csvWriter.writeRecords(data);
+    console.log(`${logPrefix2} CSV file=%s written successfully`, fileName);
+  } catch (err) {
+    if (err instanceof Error) {
+      throw new Error(format5(`error while writing csv file=%s, err: %s`, fileName, err.message));
+    }
+    throw err;
+  }
+};
+var writeGeneratorToCSV = async (generator, outputFileName) => {
+  try {
+    let rowCount = 0;
+    const firstDataRow = (await generator.next()).value;
+    if (!firstDataRow) {
+      console.log(`${logPrefix2} [%s] first data row of csv empty, skipping`, outputFileName);
+      return 0;
+    }
+    console.log(`${logPrefix2} [%s] inferring headers from first data row`, outputFileName);
+    const headers = Object.keys(firstDataRow).map((col) => {
+      return {
+        id: col,
+        title: col
+      };
+    });
+    const csvWriter = createObjectCsvWriter({
+      path: outputFileName,
+      header: headers,
+      alwaysQuote: true
+    });
+    console.log(`${logPrefix2} [%s] writing csv`, outputFileName);
+    let batch = [firstDataRow];
+    for await (const data of generator) {
+      batch.push(data);
+      rowCount++;
+      if (batch.length > 1e4) {
+        try {
+          await csvWriter.writeRecords(batch);
+        } catch (err) {
+          if (err instanceof Error) {
+            throw new Error(
+              format5(
+                `error while writing batch to csv batch (first 3 records)=%j, err: %s`,
+                batch.slice(0, 3),
+                err.message
+              )
+            );
+          }
+          throw err;
+        }
+        batch = [];
+      }
+    }
+    await csvWriter.writeRecords(batch);
+    console.log(`${logPrefix2} [%s] csv written successfully with %d rows`, outputFileName, rowCount);
+    return rowCount;
+  } catch (err) {
+    if (err instanceof Error) {
+      throw new Error(format5(`error while writing csv file=%s, err: %s`, outputFileName, err.message));
+    }
+    throw err;
+  }
+};
+
+// src/sdk/file-processing/file-stream.ts
+import XLSX2 from "xlsx";
+var FileStream = class extends Stream {
+  localFilePaths;
+  constructor(config, localFilePath, tapState, target) {
+    super(config, tapState, target);
+    this.localFilePaths = Array.isArray(localFilePath) ? localFilePath : [localFilePath];
+    this.streamId = config.streamId;
+    this.primaryKey = config.primaryKeys;
+    this.replicationMethod = config.replicationMethod;
+  }
+  /**
+   * Build JSON Schema from the file config's field definitions.
+   */
+  async getSchema() {
+    if (this.config.format === "EXCEL" /* EXCEL */) {
+      const excelConfig = this.config.excel;
+      if (excelConfig.type === "single-sheet-extraction") {
+        const jsonSchema = excelFieldsToJsonSchema(excelConfig.columns);
+        return { jsonSchema };
+      }
+      return void 0;
+    }
+    if (this.config.format === "CSV" /* CSV */) {
+      const jsonSchema = csvFieldsToJsonSchema(this.config.csv.fields);
+      return { jsonSchema };
+    }
+    throw new Error(`FileStream: No valid format config for stream ${this.streamId}`);
+  }
+  /**
+   * Yield records from all file(s).
+   * When multiple file paths are provided, records are yielded sequentially from each file.
+   */
+  async *_getRecords() {
+    for (const filePath of this.localFilePaths) {
+      if (this.config.format === "EXCEL" /* EXCEL */) {
+        const data = await extractExcelRows(filePath, this.config.excel);
+        for (const row of data) {
+          yield row;
+        }
+      } else if (this.config.format === "CSV" /* CSV */) {
+        yield* rowGeneratorFromCsv(filePath, this.config.csv);
+      } else {
+        throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
+      }
+    }
+  }
+};
+function createExcelStreamConfig(excelConfig, fileName, localFilePath) {
+  if (excelConfig.type === "single-sheet-extraction") {
+    const primaryKeys = extractPrimaryKeysFromExcelFields(excelConfig.columns);
+    let tableName2;
+    if (typeof excelConfig.tableName === "function") {
+      const variables = excelConfig.fileNameVariablesExtractor(fileName);
+      tableName2 = excelConfig.tableName(fileName, void 0, variables);
+    } else {
+      tableName2 = excelConfig.tableName;
+    }
+    return {
+      format: "EXCEL" /* EXCEL */,
+      streamId: tableName2,
+      replicationMethod: excelConfig.replicationMethod,
+      primaryKeys,
+      excel: excelConfig
+    };
+  }
+  let tableName;
+  if (typeof excelConfig.tableName === "function") {
+    if (!localFilePath) {
+      throw new Error("localFilePath is required for processor configs with dynamic table names");
+    }
+    const workbook = XLSX2.readFile(localFilePath, { dense: true });
+    const variables = excelConfig.fileNameVariablesExtractor(fileName, workbook);
+    tableName = excelConfig.tableName(fileName, workbook, variables);
+  } else {
+    tableName = excelConfig.tableName;
+  }
+  return {
+    format: "EXCEL" /* EXCEL */,
+    streamId: tableName,
+    replicationMethod: excelConfig.replicationMethod,
+    primaryKeys: [],
+    excel: excelConfig
+  };
+}
+function createCsvStreamConfig(streamId, csvConfig, options) {
+  return {
+    format: "CSV" /* CSV */,
+    streamId,
+    replicationMethod: options?.replicationMethod ?? "FULL_TABLE" /* FULL_TABLE */,
+    primaryKeys: options?.primaryKeys ?? extractPrimaryKeysFromCsvConfig(csvConfig.fields),
+    csv: csvConfig
+  };
+}
+async function processFileStreams(entries, tapState, target) {
+  for (const entry of entries) {
+    const stream = new FileStream(entry.config, entry.filePath, tapState, target);
+    await stream.sync();
+  }
+  await target.complete();
+}
+
+// src/sdk/file-processing/csv/config-builder.ts
+var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
+  _separator = ",";
+  _encoding;
+  _addSyncedAtColumn;
+  _fields;
+  _fieldsMode;
+  _replicationMethod;
+  _primaryKeys;
+  static create() {
+    return new _CsvExtractionConfigBuilder();
+  }
+  separator(sep) {
+    this._separator = sep;
+    return this;
+  }
+  encoding(enc) {
+    this._encoding = enc;
+    return this;
+  }
+  addSyncedAtColumn(enabled = true) {
+    this._addSyncedAtColumn = enabled;
+    return this;
+  }
+  fieldsFromDict(mapping) {
+    if (this._fieldsMode === "array") {
+      throw new Error("Cannot use fieldsFromDict after fieldsFromArray");
+    }
+    this._fieldsMode = "dict";
+    this._fields = mapping;
+    return this;
+  }
+  fieldsFromArray(fields) {
+    if (this._fieldsMode === "dict") {
+      throw new Error("Cannot use fieldsFromArray after fieldsFromDict");
+    }
+    this._fieldsMode = "array";
+    this._fields = fields;
+    return this;
+  }
+  replicationMethod(method) {
+    this._replicationMethod = method;
+    return this;
+  }
+  primaryKeys(keys) {
+    this._primaryKeys = keys;
+    return this;
+  }
+  build() {
+    if (!this._fields) {
+      throw new Error("Fields must be configured (use fieldsFromDict or fieldsFromArray)");
+    }
+    if (Array.isArray(this._fields) && this._fields.length === 0) {
+      throw new Error("Fields must not be empty");
+    }
+    if (!Array.isArray(this._fields) && Object.keys(this._fields).length === 0) {
+      throw new Error("Fields must not be empty");
+    }
+    const config = {
+      separator: this._separator,
+      fields: this._fields
+    };
+    if (this._encoding !== void 0) {
+      config.encoding = this._encoding;
+    }
+    if (this._addSyncedAtColumn !== void 0) {
+      config.addSyncedAtColumn = this._addSyncedAtColumn;
+    }
+    return config;
+  }
+  buildStreamConfig(streamId) {
+    const csvConfig = this.build();
+    const options = {};
+    if (this._replicationMethod !== void 0) {
+      options.replicationMethod = this._replicationMethod;
+    }
+    if (this._primaryKeys !== void 0) {
+      options.primaryKeys = this._primaryKeys;
+    }
+    return createCsvStreamConfig(streamId, csvConfig, options);
+  }
+};
+
+// src/sdk/file-processing/file-tap.ts
+var FileTap = class _FileTap extends Tap {
+  entries;
+  constructor(target, config, stateProvider, entries) {
+    super(target, config, stateProvider);
+    this.entries = entries;
+  }
+  /**
+   * Convenience constructor for the common case where all configs share the same file path.
+   */
+  static fromConfigs(target, config, stateProvider, fileConfigs, sharedFilePath) {
+    const entries = fileConfigs.map((c) => ({
+      config: c,
+      filePath: sharedFilePath
+    }));
+    return new _FileTap(target, config, stateProvider, entries);
+  }
+  async init() {
+    for (const entry of this.entries) {
+      const stream = new FileStream(
+        entry.config,
+        entry.filePath,
+        this.tapState,
+        this.target
+      );
+      this.streams.push(stream);
+    }
+  }
+};
+
+// src/sdk/file-processing/file-patterns.ts
+var VariableExtractors = class {
+  /**
+   * Returns the filename as a variable.
+   */
+  static filename() {
+    return (filename) => ({
+      fileName: filename
+    });
+  }
+  /**
+   * Extracts named capture groups from a regex pattern.
+   */
+  static regex(pattern) {
+    return (filename) => {
+      const match = filename.match(pattern);
+      return match?.groups ?? {};
+    };
+  }
+  /**
+   * Combines multiple extraction functions.
+   */
+  static combine(...extractors) {
+    return (filename) => {
+      return extractors.reduce((acc, extractor) => {
+        return { ...acc, ...extractor(filename) };
+      }, {});
+    };
+  }
+};
+var FilePatterns = class {
+  /**
+   * Creates a validator for files starting with a specific prefix (case-insensitive).
+   */
+  static startsWith(prefix) {
+    return (filename) => filename.toLowerCase().startsWith(prefix.toLowerCase());
+  }
+  /**
+   * Creates a validator for files matching a regex pattern.
+   */
+  static regex(pattern) {
+    return (filename) => pattern.test(filename);
+  }
+  /**
+   * Combines multiple validators with AND logic.
+   */
+  static and(...validators) {
+    return (filename) => validators.every((validator) => validator(filename));
+  }
+  /**
+   * Combines multiple validators with OR logic.
+   */
+  static or(...validators) {
+    return (filename) => validators.some((validator) => validator(filename));
+  }
+};
+
+// src/services/sftp.ts
+import Client from "ssh2-sftp-client";
+import { default as default2 } from "ssh2-sftp-client";
+var SftpService = new Client();
+
+// src/services/cloud-storage.ts
+import { Storage } from "@google-cloud/storage";
+import { format as format6 } from "util";
+import * as pathModule from "path";
+import { existsSync, mkdirSync, unlinkSync as unlinkSync2 } from "fs";
+
+// node_modules/uuid/dist/esm-node/rng.js
+import crypto from "crypto";
+var rnds8Pool = new Uint8Array(256);
+var poolPtr = rnds8Pool.length;
+function rng() {
+  if (poolPtr > rnds8Pool.length - 16) {
+    crypto.randomFillSync(rnds8Pool);
+    poolPtr = 0;
+  }
+  return rnds8Pool.slice(poolPtr, poolPtr += 16);
+}
+
+// node_modules/uuid/dist/esm-node/regex.js
+var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
+
+// node_modules/uuid/dist/esm-node/validate.js
+function validate(uuid) {
+  return typeof uuid === "string" && regex_default.test(uuid);
+}
+var validate_default = validate;
+
+// node_modules/uuid/dist/esm-node/stringify.js
+var byteToHex = [];
+for (let i = 0; i < 256; ++i) {
+  byteToHex.push((i + 256).toString(16).substr(1));
+}
+function stringify3(arr, offset = 0) {
+  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
+  if (!validate_default(uuid)) {
+    throw TypeError("Stringified UUID is invalid");
+  }
+  return uuid;
+}
+var stringify_default = stringify3;
+
+// node_modules/uuid/dist/esm-node/v4.js
+function v4(options, buf, offset) {
+  options = options || {};
+  const rnds = options.random || (options.rng || rng)();
+  rnds[6] = rnds[6] & 15 | 64;
+  rnds[8] = rnds[8] & 63 | 128;
+  if (buf) {
+    offset = offset || 0;
+    for (let i = 0; i < 16; ++i) {
+      buf[offset + i] = rnds[i];
+    }
+    return buf;
+  }
+  return stringify_default(rnds);
+}
+var v4_default = v4;
+
+// src/services/cloud-storage.ts
+var logPrefix3 = "[CloudStorageService]";
+var tmpDir = "tmp";
+var CloudStorageService = class {
+  storage;
+  bucket;
+  processedSuffix;
+  supportedExtensions;
+  path;
+  constructor(bucketName, path2, opts) {
+    this.storage = new Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
+    this.bucket = this.storage.bucket(bucketName);
+    this.path = path2;
+    this.processedSuffix = opts?.processedSuffix ?? ".processed";
+    this.supportedExtensions = opts?.supportedExtensions ?? [];
+  }
+  /**
+   * Lists all files in the bucket with an optional prefix.
+   */
+  async listFiles(prefix) {
+    const effectivePrefix = prefix || this.path;
+    const [files] = await this.bucket.getFiles(
+      effectivePrefix ? { prefix: effectivePrefix } : void 0
+    );
+    return files.map((f) => f.name);
+  }
+  /**
+   * Returns files that haven't been marked as processed.
+   * Filters by supported extensions if configured.
+   */
+  async getUnprocessedFiles() {
+    const allFiles = await this.listFiles();
+    const markerFiles = new Set(
+      allFiles.filter((f) => f.endsWith(this.processedSuffix)).map((f) => f.replace(this.processedSuffix, ""))
+    );
+    return allFiles.filter((f) => {
+      if (f.endsWith(this.processedSuffix)) return false;
+      if (f.endsWith("/")) return false;
+      if (this.supportedExtensions.length > 0) {
+        const ext = pathModule.extname(f).toLowerCase();
+        if (!this.supportedExtensions.includes(ext)) return false;
+      }
+      return !markerFiles.has(f);
+    });
+  }
+  /**
+   * Creates a marker file to indicate a file has been processed.
+   */
+  async createMarkerFile(fileName) {
+    const markerFileName = `${fileName}${this.processedSuffix}`;
+    try {
+      const file = this.bucket.file(markerFileName);
+      await file.save(format6("Marked file %s as processed", fileName));
+      logger.info(`${logPrefix3} Marker file ${markerFileName} created successfully.`);
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error(format6(`error while writing marker file=%s, err:%s`, markerFileName, err.message));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Downloads a file from GCS to a local tmp directory.
+   * Returns the local path of the downloaded file.
+   */
+  async downloadFile(filePath, fileName) {
+    const file = this.bucket.file(filePath);
+    const tmpDirPath = pathModule.join(process.cwd(), tmpDir);
+    if (!existsSync(tmpDirPath)) {
+      mkdirSync(tmpDirPath);
+    }
+    const destFilename = pathModule.join(process.cwd(), tmpDir, fileName);
+    try {
+      if (existsSync(destFilename)) {
+        unlinkSync2(destFilename);
+      }
+      await file.download({ destination: destFilename });
+      logger.info(`${logPrefix3} Downloaded ${fileName} to ${destFilename}`);
+      return destFilename;
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error(format6(
+          `can't download file=%s from bucket, err:%s`,
+          fileName,
+          err.message
+        ));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Uploads a local file to GCS.
+   */
+  async uploadFile(localPath, destPath) {
+    try {
+      logger.info(`${logPrefix3} preparing to upload '%s' to '%s'`, localPath, destPath);
+      const [file] = await this.bucket.upload(localPath, { destination: destPath });
+      logger.info(`${logPrefix3} file %s has been successfully uploaded into %s`, localPath, destPath);
+      return file;
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error(format6(`error while uploading file file=%s into path=%s, err:%s`, localPath, destPath, err.message));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Reads a GCS object and returns its contents as a UTF-8 string.
+   */
+  async readObjectAsString(objectPath) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const fileRef = this.bucket.file(objectPath);
+      const [exists] = await fileRef.exists();
+      if (!exists) {
+        throw new Error(`GCS object not found: gs://${this.bucket.name}/${objectPath}`);
+      }
+      const [contents] = await fileRef.download();
+      return contents.toString("utf8");
+    } catch (err) {
+      throw new Error(format6(`error reading GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+  /**
+   * Writes a string to a GCS object with JSON content type.
+   */
+  async writeStringObject(objectPath, contents) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const fileRef = this.bucket.file(objectPath);
+      await fileRef.save(contents, { contentType: "application/json" });
+      logger.info(`Uploaded object to gs://${this.bucket.name}/${objectPath}`);
+    } catch (err) {
+      throw new Error(format6(`error writing GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+  /**
+   * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+   * Returns the GCS File reference.
+   */
+  async uploadFileWithUniqueName(filePath, prefix, streamId) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const destinationFileName = `${prefix}/${streamId}-${v4_default()}.jsonnl`;
+      await this.bucket.upload(filePath, {
+        destination: destinationFileName
+      });
+      logger.info(`Uploaded ${filePath} into ${this.bucket.name}/${destinationFileName} GCS File`);
+      return this.bucket.file(destinationFileName);
+    } catch (err) {
+      logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
+
+        Error: ${err.message}
+        Stack: ${err.stack}
+        Code: ${err.code}
+        `);
+      throw err;
     }
-  });
-  returnString = returnString.replace(/^[-\d\s]*/g, "");
-  returnString = returnString.toLowerCase();
-  returnString = returnString.replace("`", "");
-  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
-  if (returnString.length > 300) {
-    returnString = returnString.substr(0, 300);
   }
-  const pattern = /[^a-zA-Z0-9_]/gm;
-  return returnString.replace(pattern, "_");
+};
+
+// src/services/zip.ts
+import * as fse from "fs-extra";
+import decompress from "decompress";
+var unzip = async (zipFilePath, extractedPath) => {
+  await fse.ensureDir(extractedPath);
+  await decompress(zipFilePath, extractedPath);
+  return extractedPath;
+};
+
+// src/targets/bigquery/helpers.ts
+var safeColumnName = (key) => {
+  let returnString = key;
+  const shouldNotStartWith = ["_TABLE_", "_FILE_", "_PARTITION"];
+  shouldNotStartWith.forEach((snm) => {
+    if (returnString.startsWith(snm)) {
+      returnString = returnString.replace(snm, "");
+    }
+  });
+  returnString = returnString.replace(/^[-\d\s]*/g, "");
+  returnString = returnString.toLowerCase();
+  returnString = returnString.replace("`", "");
+  returnString = returnString.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+  if (returnString.length > 300) {
+    returnString = returnString.substr(0, 300);
+  }
+  const pattern = /[^a-zA-Z0-9_]/gm;
+  return returnString.replace(pattern, "_");
 };
 var safeTableName = (key) => {
   let returnString = key;
@@ -2460,15 +3238,15 @@ var safeTableName = (key) => {
 
 // src/targets/bigquery/service/record.ts
 import Decimal from "decimal.js";
-import moment5 from "moment";
+import dayjs5 from "dayjs";
 var validateDateRange = (record, schema) => {
   Object.keys(schema).forEach((key) => {
-    const { format: format3 } = schema[key];
-    if (format3 === "date-time") {
+    const { format: format8 } = schema[key];
+    if (format8 === "date-time") {
       if (record[key]) {
-        const formattedDate = moment5(record[key]).format(defaultDateTimeFormat);
-        const isNotTooMuchInTheFuture = moment5(record[key]).isBefore(moment5("9999-12-31 23:59:59.999999"));
-        const isNotTooMuchInThePast = moment5(record[key]).isAfter(moment5("0001-01-01 00:00:00"));
+        const formattedDate = dayjs5(record[key]).format(defaultDateTimeFormat);
+        const isNotTooMuchInTheFuture = dayjs5(record[key]).isBefore(dayjs5("9999-12-31 23:59:59.999999"));
+        const isNotTooMuchInThePast = dayjs5(record[key]).isAfter(dayjs5("0001-01-01 00:00:00"));
         if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
           record[key] = formattedDate;
         } else {
@@ -2523,7 +3301,7 @@ var convertNumberIntoDecimal = (record, schema) => {
 };
 
 // src/targets/bigquery/service/dbSync.ts
-import moment6 from "moment";
+import dayjs6 from "dayjs";
 
 // src/targets/bigquery/service/bigquery.ts
 import { BigQuery } from "@google-cloud/bigquery";
@@ -2548,96 +3326,6 @@ var BqClientHolder = class {
   }
 };
 
-// src/targets/bigquery/service/cloudStorage.ts
-import { Storage } from "@google-cloud/storage";
-
-// node_modules/uuid/dist/esm-node/rng.js
-import crypto from "crypto";
-var rnds8Pool = new Uint8Array(256);
-var poolPtr = rnds8Pool.length;
-function rng() {
-  if (poolPtr > rnds8Pool.length - 16) {
-    crypto.randomFillSync(rnds8Pool);
-    poolPtr = 0;
-  }
-  return rnds8Pool.slice(poolPtr, poolPtr += 16);
-}
-
-// node_modules/uuid/dist/esm-node/regex.js
-var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
-
-// node_modules/uuid/dist/esm-node/validate.js
-function validate(uuid) {
-  return typeof uuid === "string" && regex_default.test(uuid);
-}
-var validate_default = validate;
-
-// node_modules/uuid/dist/esm-node/stringify.js
-var byteToHex = [];
-for (let i = 0; i < 256; ++i) {
-  byteToHex.push((i + 256).toString(16).substr(1));
-}
-function stringify3(arr, offset = 0) {
-  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
-  if (!validate_default(uuid)) {
-    throw TypeError("Stringified UUID is invalid");
-  }
-  return uuid;
-}
-var stringify_default = stringify3;
-
-// node_modules/uuid/dist/esm-node/v4.js
-function v4(options, buf, offset) {
-  options = options || {};
-  const rnds = options.random || (options.rng || rng)();
-  rnds[6] = rnds[6] & 15 | 64;
-  rnds[8] = rnds[8] & 63 | 128;
-  if (buf) {
-    offset = offset || 0;
-    for (let i = 0; i < 16; ++i) {
-      buf[offset + i] = rnds[i];
-    }
-    return buf;
-  }
-  return stringify_default(rnds);
-}
-var v4_default = v4;
-
-// src/targets/bigquery/service/cloudStorage.ts
-var uploadFileInBucket = async (streamId, gcsBuckerName, filePath) => {
-  try {
-    const retryOptions = { autoRetry: true, maxRetries: 20 };
-    let storage = new Storage({ retryOptions });
-    const bucketRef = storage.bucket(gcsBuckerName);
-    await bucketRef.get({ autoCreate: false });
-    const destinationFileName = `${streamId}-${v4_default()}.jsonnl`;
-    await bucketRef.upload(filePath, {
-      destination: destinationFileName
-    });
-    logger.info(`\u{1F5F3}  Uploaded ${filePath} into ${gcsBuckerName}/${destinationFileName} GCS File`);
-    return bucketRef.file(destinationFileName);
-  } catch (err) {
-    logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
-        
-        Error: ${err.message}
-        Stack: ${err.stack}
-        Code: ${err.code}
-        `);
-    if (err.code < 500) {
-      await haltAndCatchFire(
-        `unauthorized`,
-        `We couldn't connect to your Cloud Storage bucket \u{1F614}
-                    
-                    The error from Google is: '${err.message}' \u{1F440}
-                    
-                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
-        `Got error from cloud storage lib`
-      );
-    }
-    throw err;
-  }
-};
-
 // src/targets/bigquery/service/dbSync.ts
 import chunk from "lodash/chunk.js";
 import { Semaphore as Semaphore2 } from "async-mutex";
@@ -2678,21 +3366,21 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
   safeColumnName = safeColumnName;
   getWarehouseTypeFromJSONSchema = (definition) => {
     const type = definition.type;
-    const format3 = definition.format;
+    const format8 = definition.format;
     let typeArr;
     if (type instanceof Array) {
       typeArr = type;
     } else {
       typeArr = [type];
     }
-    if (format3) {
-      switch (format3) {
+    if (format8) {
+      switch (format8) {
         case "date-time":
           return "TIMESTAMP";
         case "json":
           return "STRING";
         default:
-          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format3}`);
+          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format8}`);
       }
     } else if (typeArr.includes("number")) {
       return "NUMERIC";
@@ -2788,6 +3476,13 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
       ...this.getMergeQueries()
     ];
   };
+  getAppendQueries = () => {
+    const columnUnsafeToSafeMapping = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    const columns = Object.keys(columnUnsafeToSafeMapping).map((unsafeKey) => `\`${columnUnsafeToSafeMapping[unsafeKey]}\``).join(`, `);
+    return [
+      `INSERT INTO ${this.sqlTableId} (${columns}) SELECT ${columns} FROM ${this.sqlStagingTableId};`
+    ];
+  };
   createDatabaseAndSchemaIfNotExists = async (retryCount) => {
     try {
       await semaphore2.runExclusive(async () => {
@@ -2892,7 +3587,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
         await this.deleteStagingArea();
       }
       const schema = this.generateBigquerySchema();
-      const expirationTime = moment6().add("1", "day").valueOf().toString();
+      const expirationTime = dayjs6().add(1, "day").valueOf().toString();
       const createTableOptions = {
         schema,
         expirationTime
@@ -2912,10 +3607,11 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
   };
   loadStreamInStagingArea = async (localFilePath) => {
     try {
-      const file = await uploadFileInBucket(
-        this.streamId,
-        this.config.loading_deck_gcs_bucket_name,
-        localFilePath
+      const gcsService = new CloudStorageService(this.config.loading_deck_gcs_bucket_name);
+      const file = await gcsService.uploadFileWithUniqueName(
+        localFilePath,
+        this.config.connector_id,
+        this.streamId
       );
       const metadata = {
         sourceFormat: "NEWLINE_DELIMITED_JSON",
@@ -2924,6 +3620,17 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
       await this.loadGCSFileInTable(this.sqlStagingTableId, file, metadata);
     } catch (err) {
       logger.error(`StreamId: ${this.streamId} - Error while uploading stream.`);
+      if (err.code < 500) {
+        await haltAndCatchFire(
+          `unauthorized`,
+          `We couldn't connect to your Cloud Storage bucket \u{1F614}
+
+                    The error from Google is: '${err.message}' \u{1F440}
+
+                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
+          `Got error from cloud storage lib`
+        );
+      }
       throw err;
     }
   };
@@ -3026,8 +3733,8 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
 
 // src/targets/bigquery/main.ts
 var BigQueryTarget = class extends ITarget {
-  constructor(config, resolver) {
-    super(config, resolver);
+  constructor(config, stateProvider) {
+    super(config, stateProvider);
     this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
   }
   static requiredConfigKeys = [
@@ -3050,15 +3757,61 @@ var BigQueryTarget = class extends ITarget {
   validateDateRange = validateDateRange;
   convertNumberIntoDecimal = convertNumberIntoDecimal;
 };
+
+// src/state-providers/gcs/main.ts
+import { format as format7 } from "util";
+var GCSStateProvider = class {
+  bucketName;
+  connectorId;
+  service;
+  constructor(connectorId, bucketName) {
+    this.connectorId = connectorId;
+    this.bucketName = bucketName;
+    this.service = new CloudStorageService(bucketName);
+  }
+  getObjectPath() {
+    return `${this.connectorId}/state.json`;
+  }
+  async getState() {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
+      const raw = await this.service.readObjectAsString(objectPath);
+      const parsed = JSON.parse(raw);
+      return { state: parsed };
+    } catch (err) {
+      logger.warn(format7(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
+      return { state: void 0 };
+    }
+  }
+  async writeState(state) {
+    const objectPath = this.getObjectPath();
+    try {
+      logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
+      await this.service.writeStringObject(objectPath, state);
+    } catch (err) {
+      logger.error(format7(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+      throw new Error(format7(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+};
 export {
   ALL_STREAM_ID_KEYWORD,
   API_CALLS_METRIC_NAME,
   Authenticator,
   BATCH_INTERVAL_MS,
   BigQueryTarget,
+  CloudStorageService,
   CounterMetric,
+  CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME,
+  ExcelExtractionConfigBuilder,
+  FileFormat,
+  FilePatterns,
+  FileStream,
+  FileTap,
+  GCSStateProvider,
   ITarget,
   MissingFieldInSchemaError,
   MissingSchemaError,
@@ -3066,22 +3819,37 @@ export {
   ROWS_SYNCED_METRIC_NAME,
   RecordMessage,
   RenameColumnStore,
+  ReplicationMethod,
   ReplicationMethodMessage,
-  Resolver,
   SchemaMessage,
   SchemaValidationError,
+  default2 as SftpClient,
+  SftpService,
   StateMessage,
   StateService,
   Stream,
-  StreamMetadata,
   StreamWarehouseSyncService,
   Tap,
+  VariableExtractors,
   _greaterThan,
   addWhalyFields,
+  checkCsvHeaderRow,
+  createCellReference,
+  createCsvStreamConfig,
+  createExcelGenerator,
+  createExcelStreamConfig,
   createTemporaryFileStream,
+  csvFieldsToJsonSchema,
+  excelColumnToIndex,
+  excelFieldsToJsonSchema,
+  excelRowToIndex,
+  extractExcelRows,
+  extractPrimaryKeysFromCsvConfig,
+  extractPrimaryKeysFromExcelFields,
   extractStateForStream,
+  fieldTypeToJsonSchema,
   finalizeStateProgressMarkers,
-  findRelatedRelationships,
+  findAllMatchingConfigs,
   flattenSchema,
   getAllMetrics,
   getAxiosInstance,
@@ -3094,16 +3862,23 @@ export {
   gracefulExit,
   haltAndCatchFire,
   incrementStreamState,
+  indexToExcelColumn,
+  indexToExcelRow,
   loadJson,
-  loadResolver,
   loadSchema,
   logger,
+  parseCellReference,
   postFormDataApiCall,
   postJSONApiCall,
   postUrlEncodedApiCall,
   printMemoryFootprint,
+  processFileStreams,
   removeParasiteProperties,
+  rowGeneratorFromCsv,
   safePath,
-  writeStateFile
+  unzip,
+  validateExcelConfig,
+  writeDataToCsv,
+  writeGeneratorToCSV
 };
 //# sourceMappingURL=index.mjs.map