@whaly/connector-sdk 0.2.0 → 0.3.1

package/dist/index.js

@@ -35,9 +35,16 @@ __export(index_exports, {
   Authenticator: () => Authenticator,
   BATCH_INTERVAL_MS: () => BATCH_INTERVAL_MS,
   BigQueryTarget: () => BigQueryTarget,
+  CloudStorageService: () => CloudStorageService,
   CounterMetric: () => CounterMetric,
+  CsvExtractionConfigBuilder: () => CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
+  ExcelExtractionConfigBuilder: () => ExcelExtractionConfigBuilder,
+  FileFormat: () => FileFormat,
+  FilePatterns: () => FilePatterns,
+  FileStream: () => FileStream,
+  FileTap: () => FileTap,
   GCSStateProvider: () => GCSStateProvider,
   ITarget: () => ITarget,
   MissingFieldInSchemaError: () => MissingFieldInSchemaError,
@@ -50,16 +57,34 @@ __export(index_exports, {
   ReplicationMethodMessage: () => ReplicationMethodMessage,
   SchemaMessage: () => SchemaMessage,
   SchemaValidationError: () => SchemaValidationError,
+  SftpClient: () => import_ssh2_sftp_client2.default,
+  SftpService: () => SftpService,
   StateMessage: () => StateMessage,
   StateService: () => StateService,
   Stream: () => Stream,
   StreamWarehouseSyncService: () => StreamWarehouseSyncService,
   Tap: () => Tap,
+  VariableExtractors: () => VariableExtractors,
   _greaterThan: () => _greaterThan,
   addWhalyFields: () => addWhalyFields,
+  checkCsvHeaderRow: () => checkCsvHeaderRow,
+  countCsvLines: () => countCsvLines,
+  createCellReference: () => createCellReference,
+  createCsvStreamConfig: () => createCsvStreamConfig,
+  createExcelGenerator: () => createExcelGenerator,
+  createExcelStreamConfig: () => createExcelStreamConfig,
   createTemporaryFileStream: () => createTemporaryFileStream,
+  csvFieldsToJsonSchema: () => csvFieldsToJsonSchema,
+  excelColumnToIndex: () => excelColumnToIndex,
+  excelFieldsToJsonSchema: () => excelFieldsToJsonSchema,
+  excelRowToIndex: () => excelRowToIndex,
+  extractExcelRows: () => extractExcelRows,
+  extractPrimaryKeysFromCsvConfig: () => extractPrimaryKeysFromCsvConfig,
+  extractPrimaryKeysFromExcelFields: () => extractPrimaryKeysFromExcelFields,
   extractStateForStream: () => extractStateForStream,
+  fieldTypeToJsonSchema: () => fieldTypeToJsonSchema,
   finalizeStateProgressMarkers: () => finalizeStateProgressMarkers,
+  findAllMatchingConfigs: () => findAllMatchingConfigs,
   flattenSchema: () => flattenSchema,
   getAllMetrics: () => getAllMetrics,
   getAxiosInstance: () => getAxiosInstance,
@@ -72,15 +97,24 @@ __export(index_exports, {
   gracefulExit: () => gracefulExit,
   haltAndCatchFire: () => haltAndCatchFire,
   incrementStreamState: () => incrementStreamState,
+  indexToExcelColumn: () => indexToExcelColumn,
+  indexToExcelRow: () => indexToExcelRow,
   loadJson: () => loadJson,
   loadSchema: () => loadSchema,
   logger: () => logger,
+  parseCellReference: () => parseCellReference,
   postFormDataApiCall: () => postFormDataApiCall,
   postJSONApiCall: () => postJSONApiCall,
   postUrlEncodedApiCall: () => postUrlEncodedApiCall,
   printMemoryFootprint: () => printMemoryFootprint,
+  processFileStreams: () => processFileStreams,
   removeParasiteProperties: () => removeParasiteProperties,
-  safePath: () => safePath
+  rowGeneratorFromCsv: () => rowGeneratorFromCsv,
+  safePath: () => safePath,
+  unzip: () => unzip,
+  validateExcelConfig: () => validateExcelConfig,
+  writeDataToCsv: () => writeDataToCsv,
+  writeGeneratorToCSV: () => writeGeneratorToCSV
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -466,6 +500,7 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
 var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
   ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
   ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
+  ReplicationMethod2["APPEND"] = "APPEND";
   return ReplicationMethod2;
 })(ReplicationMethod || {});
 
@@ -813,6 +848,8 @@ var Stream = class {
   // However, a STATE will still be emitted if the Stream is using an incremental sync type.
   isSilent = false;
   childConcurrency;
+  // Optional total row count for percentage progress logging
+  totalRows;
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
@@ -1090,6 +1127,10 @@ var Stream = class {
         metric.increment();
       });
       rowsSent += 1;
+      if (rowsSent % 1e3 === 0) {
+        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      }
     }
     const stateServiceInst = StateService.getInstance();
     const state = stateServiceInst.getBookmark(this.streamId);
@@ -1590,6 +1631,8 @@ var StreamWarehouseSyncService = class {
       mergeQueries = this.getReplaceQueries();
     } else if (replicationMethod === "INCREMENTAL") {
       mergeQueries = this.getMergeQueries();
+    } else if (replicationMethod === "APPEND") {
+      mergeQueries = this.getAppendQueries();
     } else if (replicationMethod === "LOG_BASED") {
       mergeQueries = this.getMergeQueries();
     }
@@ -1760,7 +1803,8 @@ var flattenSchema = (streamId, schema) => {
 };
 
 // src/sdk/models/target/target.ts
-var import_jsonschema = require("jsonschema");
+var import_ajv = __toESM(require("ajv"));
+var import_ajv_formats = __toESM(require("ajv-formats"));
 var import_dayjs4 = __toESM(require("dayjs"));
 
 // src/sdk/models/target/streamDbState.ts
@@ -1771,15 +1815,15 @@ var import_fs_extra = __toESM(require("fs-extra"));
 var import_uniqueId = __toESM(require("lodash/uniqueId.js"));
 var createTemporaryFileStream = (streamId) => {
   import_fs_extra.default.ensureDirSync("./tmp");
-  const path = `./tmp/${safePath(streamId)}-${(0, import_uniqueId.default)()}.tmp`;
-  const writeStream = import_fs_extra.default.createWriteStream(path, { flags: "w" });
+  const path2 = `./tmp/${safePath(streamId)}-${(0, import_uniqueId.default)()}.tmp`;
+  const writeStream = import_fs_extra.default.createWriteStream(path2, { flags: "w" });
   return {
     stream: writeStream,
-    path
+    path: path2
   };
 };
-function safePath(path) {
-  let formattedPath = path;
+function safePath(path2) {
+  let formattedPath = path2;
   formattedPath = formattedPath.replace(/^The\s+/gi, "");
   formattedPath = formattedPath.replace(/\s+(?:\(|\[)?Dis(?:c|k) (?:One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\d)+(?:\)|\])?/gi, "");
   formattedPath = formattedPath.replace(/\s+/gi, "_");
@@ -1838,6 +1882,8 @@ var StreamState = class {
   // So we need to keep track of whether we are at the first load or not
   _hasBeenLoadedYetDuringThisSync;
   replicationMethod;
+  // Pre-compiled ajv ValidateFunction — compiled once per schema, reused for every row
+  compiledValidateFn;
   constructor(streamId, dbSync, replicationMethod) {
     this.streamId = streamId;
     this.dbSync = dbSync;
@@ -1855,6 +1901,12 @@ var StreamState = class {
   getSchema() {
     return this.schema;
   }
+  setCompiledValidateFn(fn) {
+    this.compiledValidateFn = fn;
+  }
+  getCompiledValidateFn() {
+    return this.compiledValidateFn;
+  }
   getBatchDate() {
     return this.batchDate.format(defaultDateTimeFormat);
   }
@@ -1911,8 +1963,8 @@ var ITarget = class _ITarget {
   flushedState;
   renameColumnStore;
   streams;
-  // JSON Schema validator
-  validator;
+  // JSON Schema validator (ajv — pre-compiles schemas into optimized JS functions)
+  ajv;
   constructor(config, stateProvider) {
     this.config = config;
     this.stateProvider = stateProvider;
@@ -1920,7 +1972,7 @@ var ITarget = class _ITarget {
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
-    this.validator = new import_jsonschema.Validator();
+    this.ajv = (0, import_ajv_formats.default)(new import_ajv.default({ allErrors: true, strict: false }));
     this.syncTime = (0, import_dayjs4.default)();
     this.renameColumnStore = new RenameColumnStore();
   }
@@ -1979,12 +2031,14 @@ var ITarget = class _ITarget {
       if (!stream) {
         throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
       }
-      await semaphore.runExclusive(async () => {
-        const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
-        if (shouldUploadIntermediateBatch) {
-          await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
-        }
-      });
+      if (stream.getBatchedRowCount() % 1e4 === 0) {
+        await semaphore.runExclusive(async () => {
+          const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
+          if (shouldUploadIntermediateBatch) {
+            await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
+          }
+        });
+      }
       const record = message.record;
       const schema = stream.getSchema();
       if (!schema) {
@@ -1993,13 +2047,13 @@ var ITarget = class _ITarget {
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
       const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
-      const validationResult = this.validator.validate(recordWithWhalyFields, schema);
-      if (!validationResult.valid) {
+      const validateFn = stream.getCompiledValidateFn();
+      if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
-                Validation errors: ${JSON.stringify(validationResult.errors)}
+                Validation errors: ${JSON.stringify(validateFn.errors)}
 
                 Record: ${JSON.stringify(recordWithoutParasiteProperties)}
-                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validationResult.errors);
+                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validateFn.errors ?? []);
       }
       const recordWithValidDate = this.validateDateRange(recordWithWhalyFields, schema);
       const recordWithDecimal = this.convertNumberIntoDecimal(recordWithValidDate, schema);
@@ -2088,7 +2142,9 @@ var ITarget = class _ITarget {
         }
       });
       streamState.setSchema(schema);
-      if (message.keyProperties.length === 0) {
+      streamState.setCompiledValidateFn(this.ajv.compile({ type: "object", properties: schema }));
+      const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
+      if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
             Received SCHEMA message: ${JSON.stringify(message)}`);
       }
@@ -2152,7 +2208,8 @@ var ITarget = class _ITarget {
       return false;
     }
     const batchedRecords = stream.getBatchedRowCount();
-    if (stream.getReplicationMethod() === "INCREMENTAL" && batchedRecords > 1e6) {
+    const method = stream.getReplicationMethod();
+    if ((method === "INCREMENTAL" || method === "APPEND") && batchedRecords > 1e6) {
       logger.info(`\u{1F9CA} Stream=${streamId} has reached the maximum amount of batched records. We'll trigger a flush of ALL streams in the Warehouse.`);
       return true;
     } else {
@@ -2221,6 +2278,1037 @@ var ITarget = class _ITarget {
   };
 };
 
+// src/sdk/file-processing/types.ts
+var FileFormat = /* @__PURE__ */ ((FileFormat2) => {
+  FileFormat2["CSV"] = "CSV";
+  FileFormat2["EXCEL"] = "EXCEL";
+  return FileFormat2;
+})(FileFormat || {});
+
+// src/sdk/file-processing/schema-utils.ts
+function fieldTypeToJsonSchema(fieldType) {
+  switch (fieldType) {
+    case "STRING":
+      return { type: ["null", "string"] };
+    case "FLOAT":
+      return { type: ["null", "number"] };
+    case "TIMESTAMP":
+      return { type: ["null", "string"], format: "date-time" };
+    default:
+      return { type: ["null", "string"] };
+  }
+}
+function excelFieldsToJsonSchema(fields) {
+  const properties = {};
+  for (const [key, field] of Object.entries(fields)) {
+    properties[key] = fieldTypeToJsonSchema(field.type);
+  }
+  return { type: "object", properties };
+}
+function csvFieldsToJsonSchema(config) {
+  const properties = {};
+  if (Array.isArray(config)) {
+    for (const field of config) {
+      properties[field.key] = fieldTypeToJsonSchema(field.type);
+    }
+  } else {
+    for (const [, field] of Object.entries(config)) {
+      properties[field.key] = fieldTypeToJsonSchema(field.type);
+    }
+  }
+  return { type: "object", properties };
+}
+function extractPrimaryKeysFromCsvConfig(config) {
+  if (Array.isArray(config)) {
+    return config.map((f) => f.key);
+  }
+  return Object.values(config).map((f) => f.key);
+}
+function extractPrimaryKeysFromExcelFields(fields) {
+  return Object.entries(fields).filter(([, field]) => {
+    return field.primaryKey === true || field.primaryKey === true;
+  }).map(([key]) => key);
+}
+
+// src/sdk/file-processing/excel/reader.ts
+var import_xlsx = __toESM(require("xlsx"));
+var import_path = __toESM(require("path"));
+var excelColumnToIndex = (columnLetter) => {
+  const letters = columnLetter.toUpperCase();
+  let index = 0;
+  for (let i = 0; i < letters.length; i++) {
+    const charCode = letters.charCodeAt(i);
+    if (charCode < 65 || charCode > 90) {
+      throw new Error(`Invalid column letter: ${columnLetter}`);
+    }
+    index = index * 26 + (charCode - 64);
+  }
+  return index - 1;
+};
+var indexToExcelColumn = (columnIndex) => {
+  if (columnIndex < 0) {
+    throw new Error(`Invalid column index: ${columnIndex}. Must be >= 0`);
+  }
+  let result = "";
+  let index = columnIndex;
+  while (true) {
+    result = String.fromCharCode(65 + index % 26) + result;
+    index = Math.floor(index / 26);
+    if (index === 0) break;
+    index--;
+  }
+  return result;
+};
+var excelRowToIndex = (rowNumber) => {
+  const rowNum = typeof rowNumber === "string" ? parseInt(rowNumber, 10) : rowNumber;
+  if (isNaN(rowNum) || rowNum < 1) {
+    throw new Error(`Invalid row number: ${rowNumber}. Must be >= 1`);
+  }
+  return rowNum - 1;
+};
+var indexToExcelRow = (index) => {
+  if (index < 0) {
+    throw new Error(`Invalid index: ${index}. Must be >= 0`);
+  }
+  return index + 1;
+};
+var parseCellReference = (cellRef) => {
+  const match = cellRef.match(/^([A-Z]+)(\d+)$/);
+  if (!match) {
+    throw new Error(`Invalid cell reference: ${cellRef}. Expected format like "A1", "Z24", "AA100"`);
+  }
+  const [, columnLetter, rowNumber] = match;
+  return {
+    columnIndex: excelColumnToIndex(columnLetter),
+    rowIndex: excelRowToIndex(rowNumber)
+  };
+};
+var createCellReference = (columnIndex, rowIndex) => {
+  return indexToExcelColumn(columnIndex) + indexToExcelRow(rowIndex);
+};
+var extractSingleSheetRows = async (fileName, conf) => {
+  console.log("Extracting single sheet rows from file: %s", fileName);
+  const workbook = import_xlsx.default.readFile(fileName, { dense: true });
+  const sheetKeys = Object.keys(workbook.Sheets);
+  const sheet = conf.sheetName ? workbook.Sheets[conf.sheetName] : workbook.Sheets[sheetKeys[0]];
+  if (!sheet) {
+    throw new Error(`Sheet ${conf.sheetName} not found in workbook`);
+  }
+  const sheetData = sheet["!data"];
+  if (!sheetData) {
+    throw new Error(`No data in ${conf.sheetName || "first"} sheet`);
+  }
+  if (sheetData.length === 0) {
+    throw new Error(`Sheet ${conf.sheetName || "first"} is empty`);
+  }
+  const variables = conf.fileNameVariablesExtractor(fileName, workbook);
+  const data = [];
+  let rowIdx = 0;
+  const minColumnIndex = Object.values(conf.columns).reduce((min, column) => {
+    return column.column ? Math.min(min, excelColumnToIndex(column.column)) : min;
+  }, Number.MAX_SAFE_INTEGER);
+  for (const row of sheetData) {
+    if (rowIdx < conf.numberOfRowsToSkip) {
+      rowIdx++;
+      continue;
+    }
+    if (!row) {
+      console.log("Empty row at index: %s, skipped processing.", rowIdx);
+      continue;
+    }
+    if (minColumnIndex !== Number.MAX_SAFE_INTEGER && !row[minColumnIndex]?.v) {
+      console.log("stopping processing because of empty value in the first data column (idx: %s)", minColumnIndex);
+      break;
+    }
+    const rowData = Object.entries(conf.columns).reduce((acc, [key, column]) => {
+      if (column.variableName) {
+        if (!variables) {
+          throw new Error(`No variables extracted from filename, cannot extract derived field ${key}`);
+        }
+        acc[key] = variables[column.variableName] || "";
+      } else {
+        const colIndex = excelColumnToIndex(column.column);
+        acc[key] = row[colIndex]?.v || "";
+      }
+      return acc;
+    }, {});
+    data.push(rowData);
+    rowIdx++;
+  }
+  console.log("Extracted %s rows from sheet %s", data.length, conf.sheetName || "first");
+  return data;
+};
+var extractExcelRows = async (localFilePath, conf) => {
+  let data;
+  if (conf.type === "single-sheet-extraction") {
+    data = await extractSingleSheetRows(localFilePath, conf);
+  } else if (conf.type === "processor") {
+    const workbook = import_xlsx.default.readFile(localFilePath, { dense: true });
+    data = await conf.processor(workbook);
+  } else {
+    throw new Error(`Unsupported configuration type: ${conf.type}`);
+  }
+  return data;
+};
+var validateExcelConfig = (config) => {
+  if (!config.extension) {
+    throw new Error("Extension is required");
+  }
+  if (!config.tableName) {
+    throw new Error("Table name is required");
+  }
+  if (!config.fileNameValidator || typeof config.fileNameValidator !== "function") {
+    throw new Error("File name validator function is required");
+  }
+  if (!config.fileNameVariablesExtractor || typeof config.fileNameVariablesExtractor !== "function") {
+    throw new Error("File name variables extractor function is required");
+  }
+  if (config.type === "single-sheet-extraction" && (!config.columns || Object.keys(config.columns).length === 0)) {
+    throw new Error("At least one column configuration is required for single-sheet-extraction");
+  }
+  if (config.type === "single-sheet-extraction") {
+    Object.entries(config.columns).forEach(([key, column]) => {
+      if (!column.type) {
+        throw new Error(`Column ${key} must have a type`);
+      }
+      if (column.column) {
+        const colLetter = column.column;
+        if (!/^[A-Z]+$/i.test(colLetter)) {
+          throw new Error(`Invalid column letter format: ${colLetter}`);
+        }
+      } else if (column.variableName) {
+      } else {
+        throw new Error(`Column ${key} must specify either 'column' or 'variableName'`);
+      }
+    });
+    if (config.numberOfRowsToSkip === void 0 || config.numberOfRowsToSkip < 0) {
+      throw new Error("Number of rows to skip must be specified and non-negative");
+    }
+  } else if (config.type === "processor") {
+    if (!config.processor || typeof config.processor !== "function") {
+      throw new Error("Processor function is required for processor type");
+    }
+  }
+};
+var findAllMatchingConfigs = (filename, configs) => {
+  const fileInfo = import_path.default.parse(filename);
+  return configs.filter(
+    (config) => config.fileNameValidator(fileInfo.base) && config.extension === fileInfo.ext.replace(".", "")
+  );
+};
+async function* createExcelGenerator(data) {
+  for (const row of data) {
+    yield row;
+  }
+}
+
+// src/sdk/file-processing/excel/config-builder.ts
+var ExcelExtractionConfigBuilder = class _ExcelExtractionConfigBuilder {
+  config = {
+    fileNameValidator: () => true,
+    fileNameVariablesExtractor: () => ({}),
+    replicationMethod: "FULL_TABLE" /* FULL_TABLE */
+  };
+  static create() {
+    return new _ExcelExtractionConfigBuilder();
+  }
+  extension(ext) {
+    this.config.extension = ext;
+    return this;
+  }
+  tableName(name) {
+    this.config.tableName = name;
+    return this;
+  }
+  fileValidator(validator) {
+    this.config.fileNameValidator = validator;
+    return this;
+  }
+  variablesExtractor(extractor) {
+    this.config.fileNameVariablesExtractor = extractor;
+    return this;
+  }
+  replicationMethod(method) {
+    this.config.replicationMethod = method;
+    return this;
+  }
+  columns(cols) {
+    if (this.config.type === "single-sheet-extraction") {
+      this.config.columns = cols;
+    } else {
+      throw new Error("Columns can only be set for single-sheet-extraction");
+    }
+    return this;
+  }
+  singleSheet(sheetName, skipRows = 0) {
+    this.config.type = "single-sheet-extraction";
+    if (sheetName !== void 0) {
+      this.config.sheetName = sheetName;
+    }
+    this.config.numberOfRowsToSkip = skipRows;
+    return this;
+  }
+  processor(processorFn) {
+    this.config.type = "processor";
+    this.config.processor = processorFn;
+    return this;
+  }
+  build() {
+    if (!this.config.type) {
+      throw new Error("Configuration type must be specified (singleSheet or processor)");
+    }
+    if (!this.config.extension) {
+      throw new Error("Extension must be set");
+    }
+    if (!this.config.tableName) {
+      throw new Error("Table name must be set");
+    }
+    if (!this.config.replicationMethod) {
+      throw new Error("Replication method must be set");
+    }
+    if (typeof this.config.fileNameValidator !== "function") {
+      throw new Error("fileNameValidator must be a function");
+    }
+    if (typeof this.config.fileNameVariablesExtractor !== "function") {
+      throw new Error("fileNameVariablesExtractor must be a function");
+    }
+    if (this.config.type === "single-sheet-extraction") {
+      const singleSheetConfig = this.config;
+      if (!singleSheetConfig.columns || Object.keys(singleSheetConfig.columns).length === 0) {
+        throw new Error("Columns must be set with at least one column for single-sheet-extraction");
+      }
+    }
+    if (this.config.type === "processor") {
+      const processorConfig = this.config;
+      if (typeof processorConfig.processor !== "function") {
+        throw new Error("Processor function must be set for processor configs");
+      }
+    }
+    return this.config;
+  }
+};
+
+// src/sdk/file-processing/csv/reader.ts
+var import_csv_parser = __toESM(require("csv-parser"));
+var import_fs2 = require("fs");
+var import_util3 = require("util");
+var import_stream2 = require("stream");
+var iconv = __toESM(require("iconv-lite"));
+var logPrefix = "[csvReader]";
+var createStripBomTransform = () => {
+  let isFirstChunk = true;
+  return new import_stream2.Transform({
+    transform(chunk2, encoding, callback) {
+      if (isFirstChunk) {
+        isFirstChunk = false;
+        if (chunk2.length >= 3 && chunk2[0] === 239 && chunk2[1] === 187 && chunk2[2] === 191) {
+          chunk2 = chunk2.slice(3);
+        }
+      }
+      callback(null, chunk2);
+    }
+  });
+};
+var getStringHexInfo = (str) => {
+  const bytes = Buffer.from(str, "utf8");
+  const hex = bytes.toString("hex").match(/.{2}/g)?.join(" ") || "";
+  return {
+    original: str,
+    length: str.length,
+    hex,
+    trimmed: str.trim()
+  };
+};
+var formatHexComparison = (expected, actual) => {
+  const expectedInfo = getStringHexInfo(expected);
+  const actualInfo = getStringHexInfo(actual);
+  return `
+Expected: "${expected}" (${expectedInfo.length} chars) [${expectedInfo.hex}]
+Actual:   "${actual}" (${actualInfo.length} chars) [${actualInfo.hex}]`;
+};
+async function* rowGeneratorFromCsv(path2, fileConfig) {
+  const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+  const configKeys = isGeneratorConfigArray ? [] : Object.keys(fileConfig.fields);
+  const csvOptions = { separator: fileConfig.separator };
+  if (isGeneratorConfigArray) {
+    csvOptions.headers = false;
+  }
+  const csvStream = (0, import_csv_parser.default)(csvOptions);
+  const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
+  let initialReadStream;
+  let finalInputStream;
+  const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
+  if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
+    initialReadStream = (0, import_fs2.createReadStream)(path2);
+    finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
+  } else {
+    initialReadStream = (0, import_fs2.createReadStream)(path2);
+    finalInputStream = initialReadStream.pipe(createStripBomTransform());
+  }
+  finalInputStream.pipe(csvStream);
+  try {
+    for await (const row of import_stream2.Readable.from(csvStream)) {
+      if (!isGeneratorConfigArray) {
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (const key of configKeys) {
+          const keyGenerator = fileConfig.fields[key.trim()];
+          if (!keyGenerator) continue;
+          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+        }
+        yield rowData;
+      } else {
+        const rowKeys = Object.keys(row);
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (let i = 0; i < rowKeys.length; i++) {
+          const keyGenerator = fileConfig.fields[i];
+          if (!keyGenerator) continue;
+          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+        }
+        yield rowData;
+      }
+    }
+  } finally {
+    initialReadStream.destroy();
+  }
+}
+async function countCsvLines(filePath) {
+  return new Promise((resolve, reject) => {
+    let newlines = 0;
+    (0, import_fs2.createReadStream)(filePath).on("data", (chunk2) => {
+      const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
+      for (let i = 0; i < buf.length; i++) {
+        if (buf[i] === 10) newlines++;
+      }
+    }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
+  });
+}
+var checkCsvHeaderRow = async (path2, fileConfig) => {
+  return new Promise((resolve, reject) => {
+    const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
+    console.debug(`${logPrefix} CSV Expected columns:`, colsWithParsingConfig.map((col) => {
+      const info = getStringHexInfo(col);
+      return `"${info.original}" [${info.hex}]`;
+    }));
+    const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+    const csvOptions2 = { separator: fileConfig.separator };
+    if (isGeneratorConfigArray) {
+      csvOptions2.headers = false;
+    }
+    const csvStream = (0, import_csv_parser.default)(csvOptions2);
+    let initialReadStream;
+    let finalInputStream;
+    const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
+    if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
+      initialReadStream = (0, import_fs2.createReadStream)(path2);
+      finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
+    } else {
+      initialReadStream = (0, import_fs2.createReadStream)(path2);
+      finalInputStream = initialReadStream.pipe(createStripBomTransform());
+    }
+    let isFirstLine = true;
+    finalInputStream.pipe(csvStream).on("data", (row) => {
+      if (isFirstLine) {
+        let headers = [];
+        if (isGeneratorConfigArray) {
+          headers = Object.values(row).filter((r) => !!r);
+        } else {
+          headers = Object.keys(row).filter((r) => !!r);
+        }
+        console.debug(`${logPrefix} CSV Detected headers:`, headers.map((h, idx) => {
+          const info = getStringHexInfo(h);
+          return `[${idx}]: "${info.original}" [${info.hex}]`;
+        }));
+        if (isGeneratorConfigArray) {
+          colsWithParsingConfig.forEach((col, i) => {
+            if (col !== headers[i]) {
+              const hexComparison = formatHexComparison(col, headers[i] || "<undefined>");
+              console.error(`${logPrefix} CSV Header Mismatch:${hexComparison}`);
+              throw new Error((0, import_util3.format)(
+                `CSV header mismatch at position %s: expected '%s' but got '%s' in file %s`,
+                i,
+                col,
+                headers[i] || "<undefined>",
+                path2
+              ));
+            }
+          });
+        } else {
+          colsWithParsingConfig.forEach((col) => {
+            if (!headers.includes(col)) {
+              const colInfo = getStringHexInfo(col);
+              console.error(`CSV Missing column: "${col}" [${colInfo.hex}]`);
+              console.error(`Available headers:`, headers.map((h) => `"${h}" [${getStringHexInfo(h).hex}]`));
+              const trimmedMatch = headers.find((h) => h.trim() === col.trim());
+              if (trimmedMatch) {
+                console.error(`Potential trimmed match: "${trimmedMatch}" [${getStringHexInfo(trimmedMatch).hex}]`);
+              }
+              throw new Error((0, import_util3.format)(
+                `CSV missing expected column '%s' in file: %s`,
+                col,
+                path2
+              ));
+            }
+          });
+        }
+        isFirstLine = false;
+      } else {
+        initialReadStream.destroy();
+      }
+    }).on("error", (error) => {
+      reject(new Error(`Error processing CSV file: ${error.message}`));
+    });
+    initialReadStream.on("close", () => {
+      resolve();
+    });
+  });
+};
+
+// src/sdk/file-processing/csv/writer.ts
+var import_csv_writer = require("csv-writer");
+var import_util4 = require("util");
+var logPrefix2 = "[csvWriter]";
+var writeDataToCsv = async (fileName, data) => {
+  try {
+    const firstRow = data[0];
+    if (!firstRow) {
+      console.log(`${logPrefix2} No data to write to CSV file=%s`, fileName);
+      return;
+    }
+    const csvWriter = (0, import_csv_writer.createObjectCsvWriter)({
+      path: fileName,
+      header: Object.keys(firstRow).map((col) => {
+        return { id: col, title: col };
+      })
+    });
+    await csvWriter.writeRecords(data);
+    console.log(`${logPrefix2} CSV file=%s written successfully`, fileName);
+  } catch (err) {
+    if (err instanceof Error) {
+      throw new Error((0, import_util4.format)(`error while writing csv file=%s, err: %s`, fileName, err.message));
+    }
+    throw err;
+  }
+};
+var writeGeneratorToCSV = async (generator, outputFileName) => {
+  try {
+    let rowCount = 0;
+    const firstDataRow = (await generator.next()).value;
+    if (!firstDataRow) {
+      console.log(`${logPrefix2} [%s] first data row of csv empty, skipping`, outputFileName);
+      return 0;
+    }
+    console.log(`${logPrefix2} [%s] inferring headers from first data row`, outputFileName);
+    const headers = Object.keys(firstDataRow).map((col) => {
+      return {
+        id: col,
+        title: col
+      };
+    });
+    const csvWriter = (0, import_csv_writer.createObjectCsvWriter)({
+      path: outputFileName,
+      header: headers,
+      alwaysQuote: true
+    });
+    console.log(`${logPrefix2} [%s] writing csv`, outputFileName);
+    let batch = [firstDataRow];
+    for await (const data of generator) {
+      batch.push(data);
+      rowCount++;
+      if (batch.length > 1e4) {
+        try {
+          await csvWriter.writeRecords(batch);
+        } catch (err) {
+          if (err instanceof Error) {
+            throw new Error(
+              (0, import_util4.format)(
+                `error while writing batch to csv batch (first 3 records)=%j, err: %s`,
+                batch.slice(0, 3),
+                err.message
+              )
+            );
+          }
+          throw err;
+        }
+        batch = [];
+      }
+    }
+    await csvWriter.writeRecords(batch);
+    console.log(`${logPrefix2} [%s] csv written successfully with %d rows`, outputFileName, rowCount);
+    return rowCount;
+  } catch (err) {
+    if (err instanceof Error) {
+      throw new Error((0, import_util4.format)(`error while writing csv file=%s, err: %s`, outputFileName, err.message));
+    }
+    throw err;
+  }
+};
+
+// src/sdk/file-processing/file-stream.ts
+var import_xlsx2 = __toESM(require("xlsx"));
+var FileStream = class extends Stream {
+  localFilePaths;
+  constructor(config, localFilePath, tapState, target) {
+    super(config, tapState, target);
+    this.localFilePaths = Array.isArray(localFilePath) ? localFilePath : [localFilePath];
+    this.streamId = config.streamId;
+    this.primaryKey = config.primaryKeys;
+    this.replicationMethod = config.replicationMethod;
+  }
+  /**
+   * Build JSON Schema from the file config's field definitions.
+   */
+  async getSchema() {
+    if (this.config.format === "EXCEL" /* EXCEL */) {
+      const excelConfig = this.config.excel;
+      if (excelConfig.type === "single-sheet-extraction") {
+        const jsonSchema = excelFieldsToJsonSchema(excelConfig.columns);
+        return { jsonSchema };
+      }
+      return void 0;
+    }
+    if (this.config.format === "CSV" /* CSV */) {
+      const jsonSchema = csvFieldsToJsonSchema(this.config.csv.fields);
+      return { jsonSchema };
+    }
+    throw new Error(`FileStream: No valid format config for stream ${this.streamId}`);
+  }
+  /**
+   * Yield records from all file(s).
+   * When multiple file paths are provided, records are yielded sequentially from each file.
+   * Pre-counts total rows across all files to enable percentage progress logging.
+   */
+  async *_getRecords() {
+    let total = 0;
+    for (const filePath of this.localFilePaths) {
+      if (this.config.format === "EXCEL" /* EXCEL */) {
+        const data = await extractExcelRows(filePath, this.config.excel);
+        total += data.length;
+      } else if (this.config.format === "CSV" /* CSV */) {
+        total += await countCsvLines(filePath);
+      }
+    }
+    this.totalRows = total;
+    for (const filePath of this.localFilePaths) {
+      if (this.config.format === "EXCEL" /* EXCEL */) {
+        const data = await extractExcelRows(filePath, this.config.excel);
+        for (const row of data) {
+          yield row;
+        }
+      } else if (this.config.format === "CSV" /* CSV */) {
+        yield* rowGeneratorFromCsv(filePath, this.config.csv);
+      } else {
+        throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
+      }
+    }
+  }
+};
+function createExcelStreamConfig(excelConfig, fileName, localFilePath) {
+  if (excelConfig.type === "single-sheet-extraction") {
+    const primaryKeys = extractPrimaryKeysFromExcelFields(excelConfig.columns);
+    let tableName2;
+    if (typeof excelConfig.tableName === "function") {
+      const variables = excelConfig.fileNameVariablesExtractor(fileName);
+      tableName2 = excelConfig.tableName(fileName, void 0, variables);
+    } else {
+      tableName2 = excelConfig.tableName;
+    }
+    return {
+      format: "EXCEL" /* EXCEL */,
+      streamId: tableName2,
+      replicationMethod: excelConfig.replicationMethod,
+      primaryKeys,
+      excel: excelConfig
+    };
+  }
+  let tableName;
+  if (typeof excelConfig.tableName === "function") {
+    if (!localFilePath) {
+      throw new Error("localFilePath is required for processor configs with dynamic table names");
+    }
+    const workbook = import_xlsx2.default.readFile(localFilePath, { dense: true });
+    const variables = excelConfig.fileNameVariablesExtractor(fileName, workbook);
+    tableName = excelConfig.tableName(fileName, workbook, variables);
+  } else {
+    tableName = excelConfig.tableName;
+  }
+  return {
+    format: "EXCEL" /* EXCEL */,
+    streamId: tableName,
+    replicationMethod: excelConfig.replicationMethod,
+    primaryKeys: [],
+    excel: excelConfig
+  };
+}
+function createCsvStreamConfig(streamId, csvConfig, options) {
+  return {
+    format: "CSV" /* CSV */,
+    streamId,
+    replicationMethod: options?.replicationMethod ?? "FULL_TABLE" /* FULL_TABLE */,
+    primaryKeys: options?.primaryKeys ?? extractPrimaryKeysFromCsvConfig(csvConfig.fields),
+    csv: csvConfig
+  };
+}
+async function processFileStreams(entries, tapState, target) {
+  for (const entry of entries) {
+    const stream = new FileStream(entry.config, entry.filePath, tapState, target);
+    await stream.sync();
+  }
+  await target.complete();
+}
+
+// src/sdk/file-processing/csv/config-builder.ts
+var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
+  _separator = ",";
+  _encoding;
+  _addSyncedAtColumn;
+  _fields;
+  _fieldsMode;
+  _replicationMethod;
+  _primaryKeys;
+  static create() {
+    return new _CsvExtractionConfigBuilder();
+  }
+  separator(sep) {
+    this._separator = sep;
+    return this;
+  }
+  encoding(enc) {
+    this._encoding = enc;
+    return this;
+  }
+  addSyncedAtColumn(enabled = true) {
+    this._addSyncedAtColumn = enabled;
+    return this;
+  }
+  fieldsFromDict(mapping) {
+    if (this._fieldsMode === "array") {
+      throw new Error("Cannot use fieldsFromDict after fieldsFromArray");
+    }
+    this._fieldsMode = "dict";
+    this._fields = mapping;
+    return this;
+  }
+  fieldsFromArray(fields) {
+    if (this._fieldsMode === "dict") {
+      throw new Error("Cannot use fieldsFromArray after fieldsFromDict");
+    }
+    this._fieldsMode = "array";
+    this._fields = fields;
+    return this;
+  }
+  replicationMethod(method) {
+    this._replicationMethod = method;
+    return this;
+  }
+  primaryKeys(keys) {
+    this._primaryKeys = keys;
+    return this;
+  }
+  build() {
+    if (!this._fields) {
+      throw new Error("Fields must be configured (use fieldsFromDict or fieldsFromArray)");
+    }
+    if (Array.isArray(this._fields) && this._fields.length === 0) {
+      throw new Error("Fields must not be empty");
+    }
+    if (!Array.isArray(this._fields) && Object.keys(this._fields).length === 0) {
+      throw new Error("Fields must not be empty");
+    }
+    const config = {
+      separator: this._separator,
+      fields: this._fields
+    };
+    if (this._encoding !== void 0) {
+      config.encoding = this._encoding;
+    }
+    if (this._addSyncedAtColumn !== void 0) {
+      config.addSyncedAtColumn = this._addSyncedAtColumn;
+    }
+    return config;
+  }
+  buildStreamConfig(streamId) {
+    const csvConfig = this.build();
+    const options = {};
+    if (this._replicationMethod !== void 0) {
+      options.replicationMethod = this._replicationMethod;
+    }
+    if (this._primaryKeys !== void 0) {
+      options.primaryKeys = this._primaryKeys;
+    }
+    return createCsvStreamConfig(streamId, csvConfig, options);
+  }
+};
+
+// src/sdk/file-processing/file-tap.ts
+var FileTap = class _FileTap extends Tap {
+  entries;
+  constructor(target, config, stateProvider, entries) {
+    super(target, config, stateProvider);
+    this.entries = entries;
+  }
+  /**
+   * Convenience constructor for the common case where all configs share the same file path.
+   */
+  static fromConfigs(target, config, stateProvider, fileConfigs, sharedFilePath) {
+    const entries = fileConfigs.map((c) => ({
+      config: c,
+      filePath: sharedFilePath
+    }));
+    return new _FileTap(target, config, stateProvider, entries);
+  }
+  async init() {
+    for (const entry of this.entries) {
+      const stream = new FileStream(
+        entry.config,
+        entry.filePath,
+        this.tapState,
+        this.target
+      );
+      this.streams.push(stream);
+    }
+  }
+};
+
+// src/sdk/file-processing/file-patterns.ts
+var VariableExtractors = class {
+  /**
+   * Returns the filename as a variable.
+   */
+  static filename() {
+    return (filename) => ({
+      fileName: filename
+    });
+  }
+  /**
+   * Extracts named capture groups from a regex pattern.
+   */
+  static regex(pattern) {
+    return (filename) => {
+      const match = filename.match(pattern);
+      return match?.groups ?? {};
+    };
+  }
+  /**
+   * Combines multiple extraction functions.
+   */
+  static combine(...extractors) {
+    return (filename) => {
+      return extractors.reduce((acc, extractor) => {
+        return { ...acc, ...extractor(filename) };
+      }, {});
+    };
+  }
+};
+var FilePatterns = class {
+  /**
+   * Creates a validator for files starting with a specific prefix (case-insensitive).
+   */
+  static startsWith(prefix) {
+    return (filename) => filename.toLowerCase().startsWith(prefix.toLowerCase());
+  }
+  /**
+   * Creates a validator for files matching a regex pattern.
+   */
+  static regex(pattern) {
+    return (filename) => pattern.test(filename);
+  }
+  /**
+   * Combines multiple validators with AND logic.
+   */
+  static and(...validators) {
+    return (filename) => validators.every((validator) => validator(filename));
+  }
+  /**
+   * Combines multiple validators with OR logic.
+   */
+  static or(...validators) {
+    return (filename) => validators.some((validator) => validator(filename));
+  }
+};
+
+// src/services/sftp.ts
+var import_ssh2_sftp_client = __toESM(require("ssh2-sftp-client"));
+var import_ssh2_sftp_client2 = __toESM(require("ssh2-sftp-client"));
+var SftpService = new import_ssh2_sftp_client.default();
+
+// src/services/cloud-storage.ts
+var import_storage = require("@google-cloud/storage");
+var import_util5 = require("util");
+var pathModule = __toESM(require("path"));
+var import_fs3 = require("fs");
+var import_crypto = require("crypto");
+var logPrefix3 = "[CloudStorageService]";
+var tmpDir = "tmp";
+var CloudStorageService = class {
+  storage;
+  bucket;
+  processedSuffix;
+  supportedExtensions;
+  path;
+  constructor(bucketName, path2, opts) {
+    this.storage = new import_storage.Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
+    this.bucket = this.storage.bucket(bucketName);
+    this.path = path2;
+    this.processedSuffix = opts?.processedSuffix ?? ".processed";
+    this.supportedExtensions = opts?.supportedExtensions ?? [];
+  }
+  /**
+   * Lists all files in the bucket with an optional prefix.
+   */
+  async listFiles(prefix) {
+    const effectivePrefix = prefix || this.path;
+    const [files] = await this.bucket.getFiles(
+      effectivePrefix ? { prefix: effectivePrefix } : void 0
+    );
+    return files.map((f) => f.name);
+  }
+  /**
+   * Returns files that haven't been marked as processed.
+   * Filters by supported extensions if configured.
+   */
+  async getUnprocessedFiles() {
+    const allFiles = await this.listFiles();
+    const markerFiles = new Set(
+      allFiles.filter((f) => f.endsWith(this.processedSuffix)).map((f) => f.replace(this.processedSuffix, ""))
+    );
+    return allFiles.filter((f) => {
+      if (f.endsWith(this.processedSuffix)) return false;
+      if (f.endsWith("/")) return false;
+      if (this.supportedExtensions.length > 0) {
+        const ext = pathModule.extname(f).toLowerCase();
+        if (!this.supportedExtensions.includes(ext)) return false;
+      }
+      return !markerFiles.has(f);
+    });
+  }
+  /**
+   * Creates a marker file to indicate a file has been processed.
+   */
+  async createMarkerFile(fileName) {
+    const markerFileName = `${fileName}${this.processedSuffix}`;
+    try {
+      const file = this.bucket.file(markerFileName);
+      await file.save((0, import_util5.format)("Marked file %s as processed", fileName));
+      logger.info(`${logPrefix3} Marker file ${markerFileName} created successfully.`);
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error((0, import_util5.format)(`error while writing marker file=%s, err:%s`, markerFileName, err.message));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Downloads a file from GCS to a local tmp directory.
+   * Returns the local path of the downloaded file.
+   */
+  async downloadFile(filePath, fileName) {
+    const file = this.bucket.file(filePath);
+    const tmpDirPath = pathModule.join(process.cwd(), tmpDir);
+    if (!(0, import_fs3.existsSync)(tmpDirPath)) {
+      (0, import_fs3.mkdirSync)(tmpDirPath);
+    }
+    const destFilename = pathModule.join(process.cwd(), tmpDir, fileName);
+    try {
+      if ((0, import_fs3.existsSync)(destFilename)) {
+        (0, import_fs3.unlinkSync)(destFilename);
+      }
+      await file.download({ destination: destFilename });
+      logger.info(`${logPrefix3} Downloaded ${fileName} to ${destFilename}`);
+      return destFilename;
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error((0, import_util5.format)(
+          `can't download file=%s from bucket, err:%s`,
+          fileName,
+          err.message
+        ));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Uploads a local file to GCS.
+   */
+  async uploadFile(localPath, destPath) {
+    try {
+      logger.info(`${logPrefix3} preparing to upload '%s' to '%s'`, localPath, destPath);
+      const [file] = await this.bucket.upload(localPath, { destination: destPath });
+      logger.info(`${logPrefix3} file %s has been successfully uploaded into %s`, localPath, destPath);
+      return file;
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error((0, import_util5.format)(`error while uploading file file=%s into path=%s, err:%s`, localPath, destPath, err.message));
+      }
+      throw err;
+    }
+  }
+  /**
+   * Reads a GCS object and returns its contents as a UTF-8 string.
+   */
+  async readObjectAsString(objectPath) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const fileRef = this.bucket.file(objectPath);
+      const [exists] = await fileRef.exists();
+      if (!exists) {
+        throw new Error(`GCS object not found: gs://${this.bucket.name}/${objectPath}`);
+      }
+      const [contents] = await fileRef.download();
+      return contents.toString("utf8");
+    } catch (err) {
+      throw new Error((0, import_util5.format)(`error reading GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+  /**
+   * Writes a string to a GCS object with JSON content type.
+   */
+  async writeStringObject(objectPath, contents) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const fileRef = this.bucket.file(objectPath);
+      await fileRef.save(contents, { contentType: "application/json" });
+      logger.info(`Uploaded object to gs://${this.bucket.name}/${objectPath}`);
+    } catch (err) {
+      throw new Error((0, import_util5.format)(`error writing GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
+    }
+  }
+  /**
+   * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+   * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+   * or `<prefix>/default/` otherwise.
+   * Returns the GCS File reference.
+   */
+  async uploadFileWithUniqueName(filePath, prefix, streamId) {
+    try {
+      await this.bucket.get({ autoCreate: false });
+      const runFolder = process.env.RUN_ID ?? "default";
+      const destinationFileName = `${prefix}/${runFolder}/${streamId}-${(0, import_crypto.randomUUID)()}.jsonnl`;
+      await this.bucket.upload(filePath, {
+        destination: destinationFileName
+      });
+      logger.info(`Uploaded ${filePath} into ${this.bucket.name}/${destinationFileName} GCS File`);
+      return this.bucket.file(destinationFileName);
+    } catch (err) {
+      logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
+
+        Error: ${err.message}
+        Stack: ${err.stack}
+        Code: ${err.code}
+        `);
+      throw err;
+    }
+  }
+};
+
+// src/services/zip.ts
+var fse = __toESM(require("fs-extra"));
+var import_decompress = __toESM(require("decompress"));
+var unzip = async (zipFilePath, extractedPath) => {
+  await fse.ensureDir(extractedPath);
+  await (0, import_decompress.default)(zipFilePath, extractedPath);
+  return extractedPath;
+};
+
 // src/targets/bigquery/helpers.ts
 var safeColumnName = (key) => {
   let returnString = key;
@@ -2257,8 +3345,8 @@ var import_decimal = __toESM(require("decimal.js"));
 var import_dayjs5 = __toESM(require("dayjs"));
 var validateDateRange = (record, schema) => {
   Object.keys(schema).forEach((key) => {
-    const { format: format6 } = schema[key];
-    if (format6 === "date-time") {
+    const { format: format8 } = schema[key];
+    if (format8 === "date-time") {
       if (record[key]) {
         const formattedDate = (0, import_dayjs5.default)(record[key]).format(defaultDateTimeFormat);
         const isNotTooMuchInTheFuture = (0, import_dayjs5.default)(record[key]).isBefore((0, import_dayjs5.default)("9999-12-31 23:59:59.999999"));
@@ -2342,96 +3430,6 @@ var BqClientHolder = class {
   }
 };
 
-// src/targets/bigquery/service/cloudStorage.ts
-var import_storage = require("@google-cloud/storage");
-
-// node_modules/uuid/dist/esm-node/rng.js
-var import_crypto = __toESM(require("crypto"));
-var rnds8Pool = new Uint8Array(256);
-var poolPtr = rnds8Pool.length;
-function rng() {
-  if (poolPtr > rnds8Pool.length - 16) {
-    import_crypto.default.randomFillSync(rnds8Pool);
-    poolPtr = 0;
-  }
-  return rnds8Pool.slice(poolPtr, poolPtr += 16);
-}
-
-// node_modules/uuid/dist/esm-node/regex.js
-var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
-
-// node_modules/uuid/dist/esm-node/validate.js
-function validate(uuid) {
-  return typeof uuid === "string" && regex_default.test(uuid);
-}
-var validate_default = validate;
-
-// node_modules/uuid/dist/esm-node/stringify.js
-var byteToHex = [];
-for (let i = 0; i < 256; ++i) {
-  byteToHex.push((i + 256).toString(16).substr(1));
-}
-function stringify3(arr, offset = 0) {
-  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
-  if (!validate_default(uuid)) {
-    throw TypeError("Stringified UUID is invalid");
-  }
-  return uuid;
-}
-var stringify_default = stringify3;
-
-// node_modules/uuid/dist/esm-node/v4.js
-function v4(options, buf, offset) {
-  options = options || {};
-  const rnds = options.random || (options.rng || rng)();
-  rnds[6] = rnds[6] & 15 | 64;
-  rnds[8] = rnds[8] & 63 | 128;
-  if (buf) {
-    offset = offset || 0;
-    for (let i = 0; i < 16; ++i) {
-      buf[offset + i] = rnds[i];
-    }
-    return buf;
-  }
-  return stringify_default(rnds);
-}
-var v4_default = v4;
-
-// src/targets/bigquery/service/cloudStorage.ts
-var uploadFileInBucket = async (streamId, gcsBuckerName, connectorId, filePath) => {
-  try {
-    const retryOptions = { autoRetry: true, maxRetries: 20 };
-    let storage = new import_storage.Storage({ retryOptions });
-    const bucketRef = storage.bucket(gcsBuckerName);
-    await bucketRef.get({ autoCreate: false });
-    const destinationFileName = `${connectorId}/${streamId}-${v4_default()}.jsonnl`;
-    await bucketRef.upload(filePath, {
-      destination: destinationFileName
-    });
-    logger.info(`\u{1F5F3}  Uploaded ${filePath} into ${gcsBuckerName}/${destinationFileName} GCS File`);
-    return bucketRef.file(destinationFileName);
-  } catch (err) {
-    logger.error(`Issue when uploading file into GCS bucket for stream: ${streamId}
-        
-        Error: ${err.message}
-        Stack: ${err.stack}
-        Code: ${err.code}
-        `);
-    if (err.code < 500) {
-      await haltAndCatchFire(
-        `unauthorized`,
-        `We couldn't connect to your Cloud Storage bucket \u{1F614}
-                    
-                    The error from Google is: '${err.message}' \u{1F440}
-                    
-                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
-        `Got error from cloud storage lib`
-      );
-    }
-    throw err;
-  }
-};
-
 // src/targets/bigquery/service/dbSync.ts
 var import_chunk = __toESM(require("lodash/chunk.js"));
 var import_async_mutex2 = require("async-mutex");
@@ -2472,21 +3470,21 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
   safeColumnName = safeColumnName;
   getWarehouseTypeFromJSONSchema = (definition) => {
     const type = definition.type;
-    const format6 = definition.format;
+    const format8 = definition.format;
     let typeArr;
     if (type instanceof Array) {
       typeArr = type;
     } else {
       typeArr = [type];
     }
-    if (format6) {
-      switch (format6) {
+    if (format8) {
+      switch (format8) {
         case "date-time":
           return "TIMESTAMP";
         case "json":
           return "STRING";
         default:
-          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format6}`);
+          throw new Error(`StreamId: ${this.streamId} -  Unsupported format: ${format8}`);
       }
     } else if (typeArr.includes("number")) {
       return "NUMERIC";
@@ -2582,6 +3580,13 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
       ...this.getMergeQueries()
     ];
   };
+  getAppendQueries = () => {
+    const columnUnsafeToSafeMapping = this.renamedColumnStore.getUnsafeToSafeColumnMapping(this.streamId);
+    const columns = Object.keys(columnUnsafeToSafeMapping).map((unsafeKey) => `\`${columnUnsafeToSafeMapping[unsafeKey]}\``).join(`, `);
+    return [
+      `INSERT INTO ${this.sqlTableId} (${columns}) SELECT ${columns} FROM ${this.sqlStagingTableId};`
+    ];
+  };
   createDatabaseAndSchemaIfNotExists = async (retryCount) => {
     try {
       await semaphore2.runExclusive(async () => {
@@ -2706,11 +3711,11 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
   };
   loadStreamInStagingArea = async (localFilePath) => {
     try {
-      const file = await uploadFileInBucket(
-        this.streamId,
-        this.config.loading_deck_gcs_bucket_name,
+      const gcsService = new CloudStorageService(this.config.loading_deck_gcs_bucket_name);
+      const file = await gcsService.uploadFileWithUniqueName(
+        localFilePath,
         this.config.connector_id,
-        localFilePath
+        this.streamId
       );
       const metadata = {
         sourceFormat: "NEWLINE_DELIMITED_JSON",
@@ -2719,6 +3724,17 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
       await this.loadGCSFileInTable(this.sqlStagingTableId, file, metadata);
     } catch (err) {
       logger.error(`StreamId: ${this.streamId} - Error while uploading stream.`);
+      if (err.code < 500) {
+        await haltAndCatchFire(
+          `unauthorized`,
+          `We couldn't connect to your Cloud Storage bucket \u{1F614}
+
+                    The error from Google is: '${err.message}' \u{1F440}
+
+                    Could you troubleshoot your Cloud Storage configuration in Google Cloud and sync again the source? \u{1F64F}`,
+          `Got error from cloud storage lib`
+        );
+      }
       throw err;
     }
   };
@@ -2846,47 +3862,16 @@ var BigQueryTarget = class extends ITarget {
   convertNumberIntoDecimal = convertNumberIntoDecimal;
 };
 
-// src/sdk/service/gcs.ts
-var import_storage2 = require("@google-cloud/storage");
-var import_util3 = require("util");
-var getStorage = () => new import_storage2.Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
-var readObjectAsString = async (bucketName, objectPath) => {
-  try {
-    const storage = getStorage();
-    const bucketRef = storage.bucket(bucketName);
-    await bucketRef.get({ autoCreate: false });
-    const fileRef = bucketRef.file(objectPath);
-    const [exists] = await fileRef.exists();
-    if (!exists) {
-      throw new Error(`GCS object not found: gs://${bucketName}/${objectPath}`);
-    }
-    const [contents] = await fileRef.download();
-    return contents.toString("utf8");
-  } catch (err) {
-    throw new Error((0, import_util3.format)(`error reading GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
-  }
-};
-var writeStringObject = async (bucketName, objectPath, contents) => {
-  try {
-    const storage = getStorage();
-    const bucketRef = storage.bucket(bucketName);
-    await bucketRef.get({ autoCreate: false });
-    const fileRef = bucketRef.file(objectPath);
-    await fileRef.save(contents, { contentType: "application/json" });
-    logger.info(`\u{1F5F3}  Uploaded object to gs://${bucketName}/${objectPath}`);
-  } catch (err) {
-    throw new Error((0, import_util3.format)(`error writing GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
-  }
-};
-
 // src/state-providers/gcs/main.ts
-var import_util4 = require("util");
+var import_util6 = require("util");
 var GCSStateProvider = class {
   bucketName;
   connectorId;
+  service;
   constructor(connectorId, bucketName) {
     this.connectorId = connectorId;
     this.bucketName = bucketName;
+    this.service = new CloudStorageService(bucketName);
   }
   getObjectPath() {
     return `${this.connectorId}/state.json`;
@@ -2895,11 +3880,11 @@ var GCSStateProvider = class {
     const objectPath = this.getObjectPath();
     try {
       logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
-      const raw = await readObjectAsString(this.bucketName, objectPath);
+      const raw = await this.service.readObjectAsString(objectPath);
       const parsed = JSON.parse(raw);
       return { state: parsed };
     } catch (err) {
-      logger.warn((0, import_util4.format)(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
+      logger.warn((0, import_util6.format)(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
       return { state: void 0 };
     }
   }
@@ -2907,10 +3892,10 @@ var GCSStateProvider = class {
     const objectPath = this.getObjectPath();
     try {
       logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
-      await writeStringObject(this.bucketName, objectPath, state);
+      await this.service.writeStringObject(objectPath, state);
     } catch (err) {
-      logger.error((0, import_util4.format)(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
-      throw new Error((0, import_util4.format)(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+      logger.error((0, import_util6.format)(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
+      throw new Error((0, import_util6.format)(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
     }
   }
 };
@@ -2921,9 +3906,16 @@ var GCSStateProvider = class {
   Authenticator,
   BATCH_INTERVAL_MS,
   BigQueryTarget,
+  CloudStorageService,
   CounterMetric,
+  CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS,
   EXECUTION_TIME_METRIC_NAME,
+  ExcelExtractionConfigBuilder,
+  FileFormat,
+  FilePatterns,
+  FileStream,
+  FileTap,
   GCSStateProvider,
   ITarget,
   MissingFieldInSchemaError,
@@ -2936,16 +3928,34 @@ var GCSStateProvider = class {
   ReplicationMethodMessage,
   SchemaMessage,
   SchemaValidationError,
+  SftpClient,
+  SftpService,
   StateMessage,
   StateService,
   Stream,
   StreamWarehouseSyncService,
   Tap,
+  VariableExtractors,
   _greaterThan,
   addWhalyFields,
+  checkCsvHeaderRow,
+  countCsvLines,
+  createCellReference,
+  createCsvStreamConfig,
+  createExcelGenerator,
+  createExcelStreamConfig,
   createTemporaryFileStream,
+  csvFieldsToJsonSchema,
+  excelColumnToIndex,
+  excelFieldsToJsonSchema,
+  excelRowToIndex,
+  extractExcelRows,
+  extractPrimaryKeysFromCsvConfig,
+  extractPrimaryKeysFromExcelFields,
   extractStateForStream,
+  fieldTypeToJsonSchema,
   finalizeStateProgressMarkers,
+  findAllMatchingConfigs,
   flattenSchema,
   getAllMetrics,
   getAxiosInstance,
@@ -2958,14 +3968,23 @@ var GCSStateProvider = class {
   gracefulExit,
   haltAndCatchFire,
   incrementStreamState,
+  indexToExcelColumn,
+  indexToExcelRow,
   loadJson,
   loadSchema,
   logger,
+  parseCellReference,
   postFormDataApiCall,
   postJSONApiCall,
   postUrlEncodedApiCall,
   printMemoryFootprint,
+  processFileStreams,
   removeParasiteProperties,
-  safePath
+  rowGeneratorFromCsv,
+  safePath,
+  unzip,
+  validateExcelConfig,
+  writeDataToCsv,
+  writeGeneratorToCSV
 });
 //# sourceMappingURL=index.js.map