@whaly/connector-sdk 0.3.4 → 0.3.6

package/dist/index.mjs

@@ -279,13 +279,26 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
     return response.data;
   } catch (err) {
     if (err.response.status > 400) {
-      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`. 
-            
+      throw new Error(`We got a \`${err.response.status}\` status code from endpoint: \`${endpoint}\`.
+
             Response body from the server: ${JSON.stringify(err.response.data)}`);
     }
     throw err;
   }
 };
+async function collectPaginated(options) {
+  const { initialUrl, fetchPage, getRecords, getNextUrl } = options;
+  const results = [];
+  let currentUrl = initialUrl;
+  while (true) {
+    const response = await fetchPage(currentUrl);
+    results.push(...getRecords(response));
+    const nextUrl = getNextUrl(response);
+    if (nextUrl === void 0) break;
+    currentUrl = nextUrl;
+  }
+  return results;
+}
 
 // src/sdk/service/metric.ts
 var metricStore = {};
@@ -376,6 +389,157 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
   await gracefulExit(0);
 };
 
+// src/sdk/service/dryRun.ts
+function isDryRun() {
+  const val = process.env["DRY_RUN"];
+  return val !== void 0 && val !== "" && val !== "0" && val !== "false";
+}
+function getDryRunLimit() {
+  const val = process.env["DRY_RUN_LIMIT"];
+  if (!val) return void 0;
+  const n = parseInt(val, 10);
+  return isNaN(n) || n <= 0 ? void 0 : n;
+}
+
+// src/sdk/service/apiEndpoint.ts
+var ENV_VAR = "WLY_API_ENDPOINT";
+function getApiEndpoint(explicit) {
+  const endpoint = explicit ?? process.env[ENV_VAR];
+  if (!endpoint) {
+    throw new Error(
+      `No API endpoint provided. Either pass "apiEndpoint" in config or set the ${ENV_VAR} environment variable.`
+    );
+  }
+  return endpoint;
+}
+
+// src/sdk/service/serviceAccountKey.ts
+var ENV_VAR2 = "WLY_SERVICE_ACCOUNT_KEY";
+function getServiceAccountKey(explicit) {
+  const key = explicit ?? process.env[ENV_VAR2];
+  if (!key) {
+    throw new Error(
+      `No service account key provided. Either pass "serviceAccountKey" in config or set the ${ENV_VAR2} environment variable.`
+    );
+  }
+  return key;
+}
+
+// src/sdk/service/cdnId.ts
+var ENV_VAR3 = "WLY_CDN_ID";
+function getCdnId(explicit) {
+  const cdnId = explicit ?? process.env[ENV_VAR3];
+  if (!cdnId) {
+    throw new Error(
+      `No CDN ID provided. Either pass "cdnId" in config or set the ${ENV_VAR3} environment variable.`
+    );
+  }
+  return cdnId;
+}
+
+// src/sdk/service/env.ts
+import dotenv from "dotenv";
+var dotenvLoaded = false;
+function ensureDotenv() {
+  if (!dotenvLoaded) {
+    dotenv.config();
+    dotenvLoaded = true;
+  }
+}
+function requireEnv(name) {
+  ensureDotenv();
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`Missing required environment variable: ${name}`);
+  }
+  return value;
+}
+function optionalEnv(name, defaultValue) {
+  ensureDotenv();
+  return process.env[name] ?? defaultValue;
+}
+
+// src/sdk/service/concurrency.ts
+async function runWithConcurrency(tasks, concurrency) {
+  if (concurrency < 1) {
+    throw new Error(`concurrency must be >= 1, got ${concurrency}`);
+  }
+  const results = new Array(tasks.length);
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < tasks.length) {
+      const index = nextIndex++;
+      const task = tasks[index];
+      if (task) {
+        results[index] = await task();
+      }
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, tasks.length) },
+    () => worker()
+  );
+  await Promise.all(workers);
+  return results;
+}
+async function processFromAsyncIterable(iterable, processor, concurrency, onError) {
+  if (concurrency < 1) {
+    throw new Error(`concurrency must be >= 1, got ${concurrency}`);
+  }
+  const iterator = iterable[Symbol.asyncIterator]();
+  let stopped = false;
+  async function worker() {
+    while (!stopped) {
+      const { value, done } = await iterator.next();
+      if (done) break;
+      try {
+        const result = await processor(value);
+        if (result === "stop") {
+          stopped = true;
+          break;
+        }
+      } catch (err) {
+        if (onError) {
+          onError(value, err);
+        } else {
+          stopped = true;
+          throw err;
+        }
+      }
+    }
+  }
+  const workers = Array.from({ length: concurrency }, () => worker());
+  await Promise.all(workers);
+}
+
+// src/sdk/service/mime.ts
+import path from "path";
+var MIME_TYPES = {
+  ".webp": "image/webp",
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".svg": "image/svg+xml",
+  ".avif": "image/avif",
+  ".bmp": "image/bmp",
+  ".tiff": "image/tiff",
+  ".tif": "image/tiff",
+  ".ico": "image/x-icon",
+  ".pdf": "application/pdf",
+  ".json": "application/json",
+  ".csv": "text/csv",
+  ".xml": "application/xml",
+  ".zip": "application/zip",
+  ".gz": "application/gzip",
+  ".xls": "application/vnd.ms-excel",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+};
+function getMimeType(filePathOrExt) {
+  const ext = filePathOrExt.startsWith(".") ? filePathOrExt.toLowerCase() : path.extname(filePathOrExt).toLowerCase();
+  return MIME_TYPES[ext] ?? "application/octet-stream";
+}
+
 // src/sdk/models/replication.ts
 var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
   ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
@@ -673,6 +837,9 @@ var Tap = class {
       this.tapState = cloneDeep(initialState.state);
     }
     await this.init();
+    if (isDryRun()) {
+      logger.info(`[DRY_RUN] mode active \u2014 no external writes will occur`);
+    }
     logger.info(`\u{1F680} Start syncing`);
     const state = StateService.getInstance().get();
     logger.info(`\u{1F4CD} Received state: %j`, state);
@@ -966,6 +1133,7 @@ var Stream = class {
   // Private sync methods:
   async _syncRecords(parent) {
     let rowsSent = 0;
+    const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
     let recordSyncedMetrics = [];
     if (this.rowsSyncedMetricsConf.isEnabled() === true) {
       recordSyncedMetrics = getCounterMetrics(
@@ -1007,6 +1175,10 @@ var Stream = class {
         metric.increment();
       });
       rowsSent += 1;
+      if (dryRunLimit !== void 0 && rowsSent >= dryRunLimit) {
+        logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
+        break;
+      }
       if (rowsSent % 1e3 === 0) {
         const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
         logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
@@ -1698,15 +1870,15 @@ import fs2 from "fs-extra";
 import uniqueId from "lodash/uniqueId.js";
 var createTemporaryFileStream = (streamId) => {
   fs2.ensureDirSync("./tmp");
-  const path2 = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
-  const writeStream = fs2.createWriteStream(path2, { flags: "w" });
+  const path7 = `./tmp/${safePath(streamId)}-${uniqueId()}.tmp`;
+  const writeStream = fs2.createWriteStream(path7, { flags: "w" });
   return {
     stream: writeStream,
-    path: path2
+    path: path7
   };
 };
-function safePath(path2) {
-  let formattedPath = path2;
+function safePath(path7) {
+  let formattedPath = path7;
   formattedPath = formattedPath.replace(/^The\s+/gi, "");
   formattedPath = formattedPath.replace(/\s+(?:\(|\[)?Dis(?:c|k) (?:One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\d)+(?:\)|\])?/gi, "");
   formattedPath = formattedPath.replace(/\s+/gi, "_");
@@ -1832,6 +2004,8 @@ var StreamState = class {
 // src/sdk/models/target/target.ts
 import { Semaphore } from "async-mutex";
 import Bluebird3 from "bluebird";
+import fs4 from "fs-extra";
+import path2 from "path";
 var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
@@ -1918,7 +2092,7 @@ var ITarget = class _ITarget {
       if (!stream) {
         throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
       }
-      if (stream.getBatchedRowCount() % 1e4 === 0) {
+      if (stream.getBatchedRowCount() % 1e4 === 0 && !isDryRun()) {
         await semaphore.runExclusive(async () => {
           const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
           if (shouldUploadIntermediateBatch) {
@@ -2004,7 +2178,9 @@ var ITarget = class _ITarget {
             const { hasBreakingChanges, changes } = this.analyzeSchemaChanges(prevSchema, newFlattenedSchema, streamId);
             if (hasBreakingChanges) {
               logger.info(`\u{1F525} StreamId=${streamId} - Detected BREAKING schema changes: ${changes.join(", ")}. Loading batched records before applying new schema.`);
-              await _ITarget.uploadSingleStreamToWarehouse(prevStreamState);
+              if (!isDryRun()) {
+                await _ITarget.uploadSingleStreamToWarehouse(prevStreamState);
+              }
             } else {
               logger.info(`\u2705 StreamId=${streamId} - Detected NON-BREAKING schema changes: ${changes.join(", ")}. Continuing without uploading batched records.`);
             }
@@ -2028,36 +2204,41 @@ var ITarget = class _ITarget {
       streamState.setSchema(schema);
       streamState.setCompiledValidateFn(this.ajv.compile({ type: "object", properties: schema }));
       const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
-      if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
+      if (message.keyProperties.length === 0 && streamReplicationMethod === "INCREMENTAL" /* INCREMENTAL */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
             Received SCHEMA message: ${JSON.stringify(message)}`);
       }
       streamState.setKeyProperties(message.keyProperties);
-      await dbSync.updateSchemaInWarehouse(schema, message.keyProperties);
-      const {
-        database: databaseName,
-        schema: schemaName
-      } = this.config;
-      const tableName = this.genTableName(streamId);
-      const formattedRelationshipMessage = {
-        ...message,
-        keyProperties: message.keyProperties.map((k) => {
-          const translation = streamState.getDbSync().renamedColumnStore.getColumnTranslation(message.stream, k);
-          if (translation) {
-            return translation;
-          }
-          return k;
-        })
-      };
-      await Bluebird3.map(this.schemaHooks, async (hook) => {
-        const input = {
-          databaseName,
-          schemaName,
-          tableName,
-          message: formattedRelationshipMessage
+      dbSync.renamedColumnStore.computeColumnNameForStream(streamId, schema);
+      if (!isDryRun()) {
+        await dbSync.updateSchemaInWarehouse(schema, message.keyProperties);
+        const {
+          database: databaseName,
+          schema: schemaName
+        } = this.config;
+        const tableName = this.genTableName(streamId);
+        const formattedRelationshipMessage = {
+          ...message,
+          keyProperties: message.keyProperties.map((k) => {
+            const translation = streamState.getDbSync().renamedColumnStore.getColumnTranslation(message.stream, k);
+            if (translation) {
+              return translation;
+            }
+            return k;
+          })
         };
-        await hook.writeSchema(input);
-      }, { concurrency: 3 });
+        await Bluebird3.map(this.schemaHooks, async (hook) => {
+          const input = {
+            databaseName,
+            schemaName,
+            tableName,
+            message: formattedRelationshipMessage
+          };
+          await hook.writeSchema(input);
+        }, { concurrency: 3 });
+      } else {
+        logger.info(`[DRY_RUN] Stream: ${streamId} - Skipping warehouse schema update and schema hooks`);
+      }
       return Promise.resolve();
     } catch (err) {
       logger.error(`Error when handling schema event.`, err);
@@ -2078,6 +2259,29 @@ var ITarget = class _ITarget {
   };
   complete = async () => {
     try {
+      if (isDryRun()) {
+        logger.info(`[DRY_RUN] Data Extraction is complete. Copying tmp files to out/ instead of loading to warehouse.`);
+        const outDir = path2.resolve("out");
+        await fs4.ensureDir(outDir);
+        for (const streamId of Object.keys(this.streams)) {
+          const streamState = this.streams[streamId];
+          if (!streamState || streamState.getBatchedRowCount() === 0) continue;
+          const fileToLoad = streamState.getFileToLoad();
+          await new Promise((resolve, reject) => {
+            fileToLoad.stream.end(() => {
+              fileToLoad.stream.close((err) => {
+                if (err) reject(err);
+                else resolve();
+              });
+            });
+          });
+          const destPath = path2.join(outDir, `${safePath(streamId)}.ndjson`);
+          await fs4.copy(fileToLoad.path, destPath);
+          logger.info(`[DRY_RUN] Stream: ${streamId} - Copied ${streamState.getBatchedRowCount()} records to ${destPath}`);
+          await fs4.remove(fileToLoad.path);
+        }
+        return;
+      }
       logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
       await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
       return Promise.resolve();
@@ -2216,7 +2420,7 @@ function extractPrimaryKeysFromExcelFields(fields) {
 
 // src/sdk/file-processing/excel/reader.ts
 import XLSX from "xlsx";
-import path from "path";
+import path3 from "path";
 var excelColumnToIndex = (columnLetter) => {
   const letters = columnLetter.toUpperCase();
   let index = 0;
@@ -2375,7 +2579,7 @@ var validateExcelConfig = (config) => {
   }
 };
 var findAllMatchingConfigs = (filename, configs) => {
-  const fileInfo = path.parse(filename);
+  const fileInfo = path3.parse(filename);
   return configs.filter(
     (config) => config.fileNameValidator(fileInfo.base) && config.extension === fileInfo.ext.replace(".", "")
   );
@@ -2510,7 +2714,7 @@ var formatHexComparison = (expected, actual) => {
 Expected: "${expected}" (${expectedInfo.length} chars) [${expectedInfo.hex}]
 Actual:   "${actual}" (${actualInfo.length} chars) [${actualInfo.hex}]`;
 };
-async function* rowGeneratorFromCsv(path2, fileConfig) {
+async function* rowGeneratorFromCsv(path7, fileConfig) {
   const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
   const configKeys = isGeneratorConfigArray ? [] : Object.keys(fileConfig.fields);
   const csvOptions = { separator: fileConfig.separator };
@@ -2522,10 +2726,10 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   let finalInputStream;
   const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
   if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
-    initialReadStream = createReadStream(path2);
+    initialReadStream = createReadStream(path7);
     finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
   } else {
-    initialReadStream = createReadStream(path2);
+    initialReadStream = createReadStream(path7);
     finalInputStream = initialReadStream.pipe(createStripBomTransform());
   }
   finalInputStream.pipe(csvStream);
@@ -2567,7 +2771,7 @@ async function countCsvLines(filePath) {
     }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
   });
 }
-var checkCsvHeaderRow = async (path2, fileConfig) => {
+var checkCsvHeaderRow = async (path7, fileConfig) => {
   return new Promise((resolve, reject) => {
     const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
     console.debug(`${logPrefix} CSV Expected columns:`, colsWithParsingConfig.map((col) => {
@@ -2584,10 +2788,10 @@ var checkCsvHeaderRow = async (path2, fileConfig) => {
     let finalInputStream;
     const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
     if (encodingToUse && encodingToUse !== "utf-8" && encodingToUse !== "utf8" && iconv.encodingExists(encodingToUse)) {
-      initialReadStream = createReadStream(path2);
+      initialReadStream = createReadStream(path7);
       finalInputStream = initialReadStream.pipe(iconv.decodeStream(encodingToUse));
     } else {
-      initialReadStream = createReadStream(path2);
+      initialReadStream = createReadStream(path7);
       finalInputStream = initialReadStream.pipe(createStripBomTransform());
     }
     let isFirstLine = true;
@@ -2613,7 +2817,7 @@ var checkCsvHeaderRow = async (path2, fileConfig) => {
                 i,
                 col,
                 headers[i] || "<undefined>",
-                path2
+                path7
               ));
             }
           });
@@ -2630,7 +2834,7 @@ var checkCsvHeaderRow = async (path2, fileConfig) => {
               throw new Error(format4(
                 `CSV missing expected column '%s' in file: %s`,
                 col,
-                path2
+                path7
               ));
             }
           });
@@ -2656,7 +2860,7 @@ var writeDataToCsv = async (fileName, data) => {
   try {
     const firstRow = data[0];
     if (!firstRow) {
-      console.log(`${logPrefix2} No data to write to CSV file=%s`, fileName);
+      logger.info(`${logPrefix2} No data to write to CSV file=%s`, fileName);
       return;
     }
     const csvWriter = createObjectCsvWriter({
@@ -2666,7 +2870,7 @@ var writeDataToCsv = async (fileName, data) => {
       })
     });
     await csvWriter.writeRecords(data);
-    console.log(`${logPrefix2} CSV file=%s written successfully`, fileName);
+    logger.info(`${logPrefix2} CSV file=%s written successfully`, fileName);
   } catch (err) {
     if (err instanceof Error) {
       throw new Error(format5(`error while writing csv file=%s, err: %s`, fileName, err.message));
@@ -2674,15 +2878,17 @@ var writeDataToCsv = async (fileName, data) => {
     throw err;
   }
 };
-var writeGeneratorToCSV = async (generator, outputFileName) => {
+var writeGeneratorToCSV = async (generator, outputFileName, options) => {
+  const batchSize = options?.batchSize ?? 1e4;
+  const alwaysQuote = options?.alwaysQuote ?? true;
   try {
     let rowCount = 0;
     const firstDataRow = (await generator.next()).value;
     if (!firstDataRow) {
-      console.log(`${logPrefix2} [%s] first data row of csv empty, skipping`, outputFileName);
+      logger.info(`${logPrefix2} [%s] first data row of csv empty, skipping`, outputFileName);
       return 0;
     }
-    console.log(`${logPrefix2} [%s] inferring headers from first data row`, outputFileName);
+    logger.info(`${logPrefix2} [%s] inferring headers from first data row`, outputFileName);
     const headers = Object.keys(firstDataRow).map((col) => {
       return {
         id: col,
@@ -2692,14 +2898,14 @@ var writeGeneratorToCSV = async (generator, outputFileName) => {
     const csvWriter = createObjectCsvWriter({
       path: outputFileName,
       header: headers,
-      alwaysQuote: true
+      alwaysQuote
     });
-    console.log(`${logPrefix2} [%s] writing csv`, outputFileName);
+    logger.info(`${logPrefix2} [%s] writing csv`, outputFileName);
     let batch = [firstDataRow];
     for await (const data of generator) {
       batch.push(data);
       rowCount++;
-      if (batch.length > 1e4) {
+      if (batch.length >= batchSize) {
         try {
           await csvWriter.writeRecords(batch);
         } catch (err) {
@@ -2717,8 +2923,10 @@ var writeGeneratorToCSV = async (generator, outputFileName) => {
         batch = [];
       }
     }
-    await csvWriter.writeRecords(batch);
-    console.log(`${logPrefix2} [%s] csv written successfully with %d rows`, outputFileName, rowCount);
+    if (batch.length > 0) {
+      await csvWriter.writeRecords(batch);
+    }
+    logger.info(`${logPrefix2} [%s] csv written successfully with %d rows`, outputFileName, rowCount);
     return rowCount;
   } catch (err) {
     if (err instanceof Error) {
@@ -3004,9 +3212,65 @@ var FilePatterns = class {
 };
 
 // src/services/sftp.ts
+import path4 from "path";
+import fs5 from "fs-extra";
 import Client from "ssh2-sftp-client";
 import { default as default2 } from "ssh2-sftp-client";
 var SftpService = new Client();
+var logPrefix3 = "[sftp]";
+async function listFilesRecursively(client, remotePath, options) {
+  const recursive = options?.recursive ?? true;
+  const extensions = options?.extensions?.map((e) => e.toLowerCase());
+  const maxIterations = options?.maxIterations ?? 1e3;
+  const results = [];
+  const dirs = [remotePath];
+  let iterations = 0;
+  while (dirs.length > 0) {
+    if (iterations++ >= maxIterations) {
+      logger.warn(`${logPrefix3} listFilesRecursively reached maxIterations (${maxIterations}), stopping.`);
+      break;
+    }
+    const currentDir = dirs.shift();
+    let entries;
+    try {
+      entries = await client.list(currentDir);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      logger.warn(`${logPrefix3} Failed to list ${currentDir}: ${msg}`);
+      continue;
+    }
+    for (const entry of entries) {
+      if (entry.type === "d") {
+        if (recursive && entry.name !== "." && entry.name !== "..") {
+          dirs.push(path4.posix.join(currentDir, entry.name));
+        }
+        continue;
+      }
+      if (entry.type !== "-") continue;
+      if (extensions) {
+        const ext = path4.extname(entry.name).toLowerCase();
+        if (!extensions.includes(ext)) continue;
+      }
+      const fullPath = path4.posix.join(currentDir, entry.name);
+      results.push({ ...entry, name: fullPath });
+    }
+  }
+  logger.info(`${logPrefix3} listFilesRecursively found ${results.length} files under ${remotePath}`);
+  return results;
+}
+async function downloadFiles(client, files, localDir, options) {
+  const concurrency = options?.concurrency ?? 5;
+  await fs5.ensureDir(localDir);
+  const tasks = files.map((file, index) => () => {
+    const localName = `${index}_${path4.basename(file.name)}`;
+    const localPath = path4.join(localDir, localName);
+    logger.debug(`${logPrefix3} Downloading ${file.name} \u2192 ${localPath}`);
+    return client.fastGet(file.name, localPath).then(() => localPath);
+  });
+  const results = await runWithConcurrency(tasks, concurrency);
+  logger.info(`${logPrefix3} Downloaded ${results.length} files to ${localDir}`);
+  return results;
+}
 
 // src/services/cloud-storage.ts
 import { Storage } from "@google-cloud/storage";
@@ -3014,7 +3278,7 @@ import { format as format6 } from "util";
 import * as pathModule from "path";
 import { existsSync, mkdirSync, unlinkSync as unlinkSync2 } from "fs";
 import { randomUUID } from "crypto";
-var logPrefix3 = "[CloudStorageService]";
+var logPrefix4 = "[CloudStorageService]";
 var tmpDir = "tmp";
 var CloudStorageService = class {
   storage;
@@ -3022,10 +3286,10 @@ var CloudStorageService = class {
   processedSuffix;
   supportedExtensions;
   path;
-  constructor(bucketName, path2, opts) {
+  constructor(bucketName, path7, opts) {
     this.storage = new Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
     this.bucket = this.storage.bucket(bucketName);
-    this.path = path2;
+    this.path = path7;
     this.processedSuffix = opts?.processedSuffix ?? ".processed";
     this.supportedExtensions = opts?.supportedExtensions ?? [];
   }
@@ -3066,7 +3330,7 @@ var CloudStorageService = class {
     try {
       const file = this.bucket.file(markerFileName);
       await file.save(format6("Marked file %s as processed", fileName));
-      logger.info(`${logPrefix3} Marker file ${markerFileName} created successfully.`);
+      logger.info(`${logPrefix4} Marker file ${markerFileName} created successfully.`);
     } catch (err) {
       if (err instanceof Error) {
         throw new Error(format6(`error while writing marker file=%s, err:%s`, markerFileName, err.message));
@@ -3090,7 +3354,7 @@ var CloudStorageService = class {
         unlinkSync2(destFilename);
       }
       await file.download({ destination: destFilename });
-      logger.info(`${logPrefix3} Downloaded ${fileName} to ${destFilename}`);
+      logger.info(`${logPrefix4} Downloaded ${fileName} to ${destFilename}`);
       return destFilename;
     } catch (err) {
       if (err instanceof Error) {
@@ -3108,9 +3372,9 @@ var CloudStorageService = class {
    */
   async uploadFile(localPath, destPath) {
     try {
-      logger.info(`${logPrefix3} preparing to upload '%s' to '%s'`, localPath, destPath);
+      logger.info(`${logPrefix4} preparing to upload '%s' to '%s'`, localPath, destPath);
       const [file] = await this.bucket.upload(localPath, { destination: destPath });
-      logger.info(`${logPrefix3} file %s has been successfully uploaded into %s`, localPath, destPath);
+      logger.info(`${logPrefix4} file %s has been successfully uploaded into %s`, localPath, destPath);
       return file;
     } catch (err) {
       if (err instanceof Error) {
@@ -3186,6 +3450,376 @@ var unzip = async (zipFilePath, extractedPath) => {
   return extractedPath;
 };
 
+// src/services/cdn.ts
+import axios3 from "axios";
+import axiosRetry2 from "axios-retry";
+var logPrefix5 = "[CdnService]";
+var MAX_RETRIES = 10;
+var CdnService = class {
+  axiosClient;
+  constructor(config) {
+    const resolvedKey = getServiceAccountKey(config.serviceAccountKey);
+    const resolvedEndpoint = getApiEndpoint(config.apiEndpoint);
+    this.axiosClient = axios3.create({
+      baseURL: resolvedEndpoint,
+      timeout: 12e4,
+      headers: {
+        Authorization: `Bearer ${resolvedKey}`,
+        Accept: "application/json"
+      }
+    });
+    axiosRetry2(this.axiosClient, {
+      retries: MAX_RETRIES,
+      retryCondition: (error) => {
+        const status = error.response?.status ?? 0;
+        return !error.response || status === 429 || status >= 500 && status < 600;
+      },
+      retryDelay: (retryCount, error) => {
+        const retryAfter = error.response?.headers?.["retry-after"];
+        if (retryAfter) {
+          const seconds = Number(retryAfter);
+          if (!isNaN(seconds) && seconds > 0) {
+            logger.info(`${logPrefix5} Rate limited \u2014 waiting ${seconds}s (Retry-After header), attempt ${retryCount}/${MAX_RETRIES}`);
+            return seconds * 1e3;
+          }
+        }
+        const delay = axiosRetry2.exponentialDelay(retryCount);
+        logger.info(`${logPrefix5} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
+        return delay;
+      }
+    });
+  }
+  /**
+   * Returns metadata for a CDN file via a HEAD request.
+   * If the file does not exist, returns `{ exists: false, lastModified: null, contentType: null }`.
+   * Falls back to not-existing if HEAD is not supported (405).
+   */
+  async getFileMetadata(cdnId, fileName) {
+    const url = `/v1/cdn/${cdnId}/files/${encodeURIComponent(fileName)}`;
+    try {
+      const response = await this.axiosClient.head(url);
+      const lastModifiedHeader = response.headers["last-modified"];
+      const contentTypeHeader = response.headers["content-type"];
+      return {
+        exists: true,
+        lastModified: lastModifiedHeader ? new Date(lastModifiedHeader) : null,
+        contentType: contentTypeHeader ?? null
+      };
+    } catch (err) {
+      if (axios3.isAxiosError(err)) {
+        const status = err.response?.status;
+        if (status === 404) return { exists: false, lastModified: null, contentType: null };
+        if (status === 405) {
+          logger.warn(`${logPrefix5} HEAD not supported for ${url}, treating file as absent`);
+          return { exists: false, lastModified: null, contentType: null };
+        }
+        this.throwMeaningfulError(err);
+      }
+      throw err;
+    }
+  }
+  /**
+   * Checks whether a file already exists in the CDN via a HEAD request.
+   * Returns true if the server responds 2xx, false on 404.
+   * Falls back to false (treat as not existing) if HEAD is not supported (405).
+   * No body is transferred.
+   */
+  async fileExists(cdnId, fileName) {
+    return (await this.getFileMetadata(cdnId, fileName)).exists;
+  }
+  /**
+   * Uploads a file buffer to the Whaly CDN.
+   * The PUT is idempotent — uploading the same fileName overwrites the existing file.
+   */
+  async uploadFile(cdnId, fileName, fileBuffer) {
+    const url = `/v1/cdn/${cdnId}/files/${encodeURIComponent(fileName)}`;
+    try {
+      logger.info(`${logPrefix5} Uploading ${fileName} (${fileBuffer.length} bytes) to CDN ${cdnId}`);
+      const form = new FormData();
+      form.append("file", new Blob([fileBuffer]), fileName);
+      const response = await this.axiosClient.put(url, form);
+      logger.info(`${logPrefix5} Successfully uploaded ${fileName} (status ${response.status})`);
+      return {
+        filePath: response.data?.filePath ?? `/org/${cdnId}/file/${fileName}`
+      };
+    } catch (err) {
+      if (axios3.isAxiosError(err)) {
+        this.throwMeaningfulError(err);
+      }
+      throw err;
+    }
+  }
+  throwMeaningfulError(err) {
+    const status = err.response?.status;
+    if (status === 401 || status === 403) {
+      throw new Error("Unauthorized - invalid or expired service account key");
+    } else if (status === 404) {
+      throw new Error("Organization or CDN not found");
+    } else if (status === 500) {
+      throw new Error("API server error - check CDN ID and service account permissions");
+    } else {
+      throw new Error(`API error: ${err.response?.statusText ?? err.message}`);
+    }
+  }
+};
+
+// src/sdk/models/asset-tap/asset-stream.ts
+var AssetStream = class {
+  streamId;
+  config;
+  replicationMode;
+  constructor(streamId, config, replicationMode = "INCREMENTAL") {
+    this.streamId = streamId;
+    this.config = config;
+    this.replicationMode = replicationMode;
+  }
+  async transformFile(downloadedPath, _entry) {
+    return downloadedPath;
+  }
+};
+
+// src/sdk/models/asset-tap/asset-tap.ts
+import fs6 from "fs-extra";
+import path5 from "path";
+var logPrefix6 = "[AssetTap]";
+function inferContentType(filePath, fallback) {
+  const mime = getMimeType(filePath);
+  return mime !== "application/octet-stream" ? mime : fallback;
+}
+function deriveManifestMode(streams) {
+  const modes = new Set(streams.map((s) => s.replicationMode));
+  if (modes.size > 1) {
+    logger.warn(`${logPrefix6} Streams have mixed replication modes (${[...modes].join(", ")}). Manifest will report "FULL".`);
+    return "FULL";
+  }
+  return streams[0]?.replicationMode ?? "INCREMENTAL";
+}
+var AssetTap = class {
+  config;
+  outputDir;
+  streams = [];
+  concurrency;
+  target;
+  constructor(target, config, outputDir = "out", concurrency = 5) {
+    this.target = target;
+    this.config = config;
+    this.outputDir = outputDir;
+    this.concurrency = concurrency;
+  }
+  async sync() {
+    await this.init();
+    const dryRun = isDryRun();
+    const dryRunLimit = dryRun ? getDryRunLimit() : void 0;
+    if (dryRun) {
+      logger.info(`${logPrefix6} [DRY_RUN] mode active \u2014 skipping CDN checks and uploads`);
+      await fs6.emptyDir(this.outputDir);
+      if (dryRunLimit !== void 0) {
+        logger.info(`${logPrefix6} [DRY_RUN] Limit: ${dryRunLimit} assets per stream`);
+      }
+    } else if (process.env["DRY_RUN_LIMIT"] !== void 0) {
+      logger.warn(`${logPrefix6} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
+    }
+    const tmpDir2 = path5.join(this.outputDir, "tmp");
+    await fs6.ensureDir(tmpDir2);
+    const streamManifests = [];
+    let entryIndex = 0;
+    let totalSummary = { total: 0, uploaded: 0, skipped: 0, errors: 0 };
+    for (const stream of this.streams) {
+      logger.info(`${logPrefix6} Processing stream: ${stream.streamId} (mode=${stream.replicationMode}, concurrency=${this.concurrency})`);
+      const assetEntries = [];
+      let streamAssetCount = 0;
+      await processFromAsyncIterable(
+        stream.listAssets(),
+        async (entry) => {
+          if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
+            return "stop";
+          }
+          logger.debug(`${logPrefix6} Processing entry: ${entry.sourcePath}`);
+          if (stream.replicationMode === "INCREMENTAL" && !dryRun) {
+            const shouldSync = await this.target.shouldSync(entry);
+            if (!shouldSync) {
+              logger.debug(`${logPrefix6} Skipping ${entry.sourcePath} (up-to-date)`);
+              assetEntries.push({
+                sourcePath: entry.sourcePath,
+                destinationPath: entry.destinationPath,
+                downloadedPath: "",
+                transformedPath: "",
+                size: 0,
+                contentType: entry.contentType,
+                status: "skipped",
+                transformed: false
+              });
+              return "continue";
+            }
+          }
+          const fileName = `${entryIndex}_${path5.basename(entry.sourcePath)}`;
+          const downloadedPath = path5.join(tmpDir2, fileName);
+          let uploadPath = downloadedPath;
+          entryIndex++;
+          streamAssetCount++;
+          try {
+            await stream.downloadEntry(entry, downloadedPath);
+            uploadPath = await stream.transformFile(downloadedPath, entry);
+            const wasTransformed = uploadPath !== downloadedPath;
+            const stat = await fs6.stat(uploadPath);
+            const processed = {
+              entry,
+              downloadedPath,
+              uploadPath,
+              wasTransformed,
+              size: stat.size,
+              contentType: wasTransformed ? inferContentType(uploadPath, entry.contentType) : entry.contentType
+            };
+            if (!dryRun) {
+              await this.target.uploadAsset(processed);
+            } else {
+              const inspectPath = path5.join(this.outputDir, stream.streamId, entry.destinationPath);
+              await fs6.ensureDir(path5.dirname(inspectPath));
+              await fs6.copy(uploadPath, inspectPath);
+            }
+            assetEntries.push({
+              sourcePath: entry.sourcePath,
+              destinationPath: entry.destinationPath,
+              downloadedPath,
+              transformedPath: wasTransformed ? uploadPath : "",
+              size: processed.size,
+              contentType: processed.contentType,
+              status: "uploaded",
+              transformed: wasTransformed
+            });
+            logger.info(`${logPrefix6} ${dryRun ? "[DRY_RUN] Processed" : "Uploaded"} ${entry.sourcePath} \u2192 ${entry.destinationPath}`);
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            logger.error(`${logPrefix6} Failed to process ${entry.sourcePath}: ${message}`);
+            assetEntries.push({
+              sourcePath: entry.sourcePath,
+              destinationPath: entry.destinationPath,
+              downloadedPath: "",
+              transformedPath: "",
+              size: 0,
+              contentType: entry.contentType,
+              status: "error",
+              transformed: false,
+              error: message
+            });
+          } finally {
+            await fs6.remove(downloadedPath).catch(() => void 0);
+            if (uploadPath !== downloadedPath) {
+              await fs6.remove(uploadPath).catch(() => void 0);
+            }
+          }
+          if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
+            logger.info(`${logPrefix6} [DRY_RUN] Reached limit of ${dryRunLimit} for stream "${stream.streamId}", stopping.`);
+            return "stop";
+          }
+          return "continue";
+        },
+        this.concurrency
+      );
+      const streamSummary = {
+        total: assetEntries.length,
+        uploaded: assetEntries.filter((a) => a.status === "uploaded").length,
+        skipped: assetEntries.filter((a) => a.status === "skipped").length,
+        errors: assetEntries.filter((a) => a.status === "error").length
+      };
+      totalSummary.total += streamSummary.total;
+      totalSummary.uploaded += streamSummary.uploaded;
+      totalSummary.skipped += streamSummary.skipped;
+      totalSummary.errors += streamSummary.errors;
+      streamManifests.push({
+        streamId: stream.streamId,
+        mode: stream.replicationMode,
+        syncedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        assets: assetEntries,
+        summary: streamSummary
+      });
+    }
+    const manifest = {
+      syncedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      mode: deriveManifestMode(this.streams),
+      streams: streamManifests,
+      summary: totalSummary
+    };
+    await fs6.ensureDir(this.outputDir);
+    await fs6.writeJson(path5.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
+    if (!dryRun) {
+      await this.target.complete();
+    }
+    logger.info(`${logPrefix6} Sync complete. Uploaded=${totalSummary.uploaded} Skipped=${totalSummary.skipped} Errors=${totalSummary.errors}`);
+    return manifest;
+  }
+};
+
+// src/sdk/models/asset-tap/image-transform.ts
+import sharp from "sharp";
+import path6 from "path";
+var SHARP_GRAVITY = {
+  center: "centre",
+  top: "north",
+  bottom: "south",
+  left: "west",
+  right: "east",
+  "top left": "northwest",
+  "top right": "northeast",
+  "bottom left": "southwest",
+  "bottom right": "southeast"
+};
+function parseBackground(bg) {
+  if (bg === void 0 || bg === "none" || bg === "transparent") {
+    return { r: 0, g: 0, b: 0, alpha: 0 };
+  }
+  return bg;
+}
+var ImageTransform = class {
+  /**
+   * Converts an image to WebP format using sharp (libvips).
+   *
+   * The output file is written to the same directory as the input,
+   * with the extension replaced by `.webp`.
+   *
+   * @param inputPath  Path to the source image file.
+   * @param options    Optional resize / padding parameters.
+   * @returns          Path to the produced `.webp` file.
+   */
+  static async toWebp(inputPath, options) {
+    const dir = path6.dirname(inputPath);
+    const basename = path6.basename(inputPath, path6.extname(inputPath));
+    const outputPath = path6.join(dir, `${basename}.webp`);
+    let pipeline = sharp(inputPath);
+    const hasSize = options.width !== void 0 && options.height !== void 0;
+    if (hasSize) {
+      const position = options.gravity !== void 0 ? SHARP_GRAVITY[options.gravity] ?? options.gravity : void 0;
+      if (options.extent === true) {
+        pipeline = pipeline.resize(options.width, options.height, {
+          fit: "contain",
+          position,
+          background: parseBackground(options.background)
+        });
+      } else {
+        pipeline = pipeline.resize(options.width, options.height, {
+          fit: "inside",
+          position
+        });
+      }
+    }
+    pipeline = pipeline.webp(
+      options.quality !== void 0 ? { quality: options.quality } : {}
+    );
+    await pipeline.toFile(outputPath);
+    return outputPath;
+  }
+};
+
+// src/sdk/models/asset-target/asset-target.ts
+var AssetTarget = class {
+  config;
+  constructor(config) {
+    this.config = config;
+  }
+  async complete() {
+  }
+};
+
 // src/targets/bigquery/helpers.ts
 var safeColumnName = (key) => {
   let returnString = key;
@@ -3222,7 +3856,9 @@ import Decimal from "decimal.js";
 import dayjs5 from "dayjs";
 var validateDateRange = (record, schema) => {
   Object.keys(schema).forEach((key) => {
-    const { format: format8 } = schema[key];
+    const field = schema[key];
+    if (!field) return;
+    const { format: format8 } = field;
     if (format8 === "date-time") {
       if (record[key]) {
         const formattedDate = dayjs5(record[key]).format(defaultDateTimeFormat);
@@ -3452,6 +4088,12 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
     });
   };
   getReplaceQueries = () => {
+    if (this.primaryKeys.length === 0) {
+      return [
+        `TRUNCATE TABLE ${this.sqlTableId};`,
+        ...this.getAppendQueries()
+      ];
+    }
     return [
       `TRUNCATE TABLE ${this.sqlTableId};`,
       ...this.getMergeQueries()
@@ -3739,6 +4381,46 @@ var BigQueryTarget = class extends ITarget {
   convertNumberIntoDecimal = convertNumberIntoDecimal;
 };
 
+// src/targets/cdn/main.ts
+import fs7 from "fs-extra";
+var logPrefix7 = "[CdnAssetTarget]";
+var CdnAssetTarget = class extends AssetTarget {
+  cdnService;
+  constructor(config) {
+    super(config);
+    this.cdnService = new CdnService(config);
+  }
+  async shouldSync(entry) {
+    const metadata = await this.cdnService.getFileMetadata(getCdnId(this.config.cdnId), entry.destinationPath);
+    if (!metadata.exists) {
+      logger.debug(`${logPrefix7} ${entry.destinationPath} not in CDN \u2192 will sync`);
+      return true;
+    }
+    if (entry.lastModified === void 0) {
+      logger.debug(`${logPrefix7} ${entry.destinationPath} source has no lastModified \u2192 will sync`);
+      return true;
+    }
+    if (metadata.lastModified === null) {
+      logger.debug(`${logPrefix7} ${entry.destinationPath} CDN has no lastModified \u2192 will sync`);
+      return true;
+    }
+    const sourceIsNewer = entry.lastModified > metadata.lastModified;
+    if (!sourceIsNewer) {
+      logger.debug(`${logPrefix7} Skipping ${entry.destinationPath} (CDN is up-to-date)`);
+    }
+    return sourceIsNewer;
+  }
+  async uploadAsset(asset) {
+    const fileBuffer = await fs7.readFile(asset.uploadPath);
+    await this.cdnService.uploadFile(
+      getCdnId(this.config.cdnId),
+      asset.entry.destinationPath,
+      fileBuffer
+    );
+    logger.info(`${logPrefix7} Uploaded ${asset.entry.destinationPath} (${fileBuffer.length} bytes)`);
+  }
+};
+
 // src/state-providers/gcs/main.ts
 import { format as format7 } from "util";
 var GCSStateProvider = class {
@@ -3779,9 +4461,14 @@ var GCSStateProvider = class {
 export {
   ALL_STREAM_ID_KEYWORD,
   API_CALLS_METRIC_NAME,
+  AssetStream,
+  AssetTap,
+  AssetTarget,
   Authenticator,
   BATCH_INTERVAL_MS,
   BigQueryTarget,
+  CdnAssetTarget,
+  CdnService,
   CloudStorageService,
   CounterMetric,
   CsvExtractionConfigBuilder,
@@ -3795,6 +4482,8 @@ export {
   FileTap,
   GCSStateProvider,
   ITarget,
+  ImageTransform,
+  MIME_TYPES,
   MissingFieldInSchemaError,
   MissingSchemaError,
   RESTStream,
@@ -3816,6 +4505,7 @@ export {
   _greaterThan,
   addWhalyFields,
   checkCsvHeaderRow,
+  collectPaginated,
   countCsvLines,
   createCellReference,
   createCsvStreamConfig,
@@ -3823,6 +4513,7 @@ export {
   createExcelStreamConfig,
   createTemporaryFileStream,
   csvFieldsToJsonSchema,
+  downloadFiles,
   excelColumnToIndex,
   excelFieldsToJsonSchema,
   excelRowToIndex,
@@ -3835,29 +4526,40 @@ export {
   findAllMatchingConfigs,
   flattenSchema,
   getAllMetrics,
+  getApiEndpoint,
   getAxiosInstance,
+  getCdnId,
   getCounterMetric,
   getCounterMetrics,
   getDownloadFileApiCall,
+  getDryRunLimit,
   getJSONApiCall,
   getJSONApiCallWithFullResponse,
+  getMimeType,
   getRawJSONApiCall,
+  getServiceAccountKey,
   gracefulExit,
   haltAndCatchFire,
   incrementStreamState,
   indexToExcelColumn,
   indexToExcelRow,
+  isDryRun,
+  listFilesRecursively,
   loadJson,
   loadSchema,
   logger,
+  optionalEnv,
   parseCellReference,
   postFormDataApiCall,
   postJSONApiCall,
   postUrlEncodedApiCall,
   printMemoryFootprint,
   processFileStreams,
+  processFromAsyncIterable,
   removeParasiteProperties,
+  requireEnv,
   rowGeneratorFromCsv,
+  runWithConcurrency,
   safePath,
   unzip,
   validateExcelConfig,