@whaly/connector-sdk 0.3.10 → 0.3.12

package/dist/index.js

@@ -8830,7 +8830,7 @@ var require_mime_types = __commonJS({
   "node_modules/mime-types/index.js"(exports2) {
     "use strict";
     var db = require_mime_db();
-    var extname2 = require("path").extname;
+    var extname3 = require("path").extname;
     var EXTRACT_TYPE_REGEXP = /^\s*([^;\s]*)(?:;|\s|$)/;
     var TEXT_TYPE_REGEXP = /^text\//i;
     exports2.charset = charset;
@@ -8884,7 +8884,7 @@ var require_mime_types = __commonJS({
       if (!path8 || typeof path8 !== "string") {
         return false;
       }
-      var extension2 = extname2("x." + path8).toLowerCase().substr(1);
+      var extension2 = extname3("x." + path8).toLowerCase().substr(1);
       if (!extension2) {
         return false;
       }
@@ -10003,7 +10003,7 @@ var require_form_data = __commonJS({
     var http = require("http");
     var https = require("https");
     var parseUrl = require("url").parse;
-    var fs11 = require("fs");
+    var fs12 = require("fs");
     var Stream2 = require("stream").Stream;
     var crypto = require("crypto");
     var mime = require_mime_types();
@@ -10070,7 +10070,7 @@ var require_form_data = __commonJS({
         if (value.end != void 0 && value.end != Infinity && value.start != void 0) {
           callback(null, value.end + 1 - (value.start ? value.start : 0));
         } else {
-          fs11.stat(value.path, function(err, stat) {
+          fs12.stat(value.path, function(err, stat) {
             if (err) {
               callback(err);
               return;
@@ -10342,6 +10342,7 @@ __export(index_exports, {
   GCSStateProvider: () => GCSStateProvider,
   ITarget: () => ITarget,
   ImageTransform: () => ImageTransform,
+  LocalStorageService: () => LocalStorageService,
   MIME_TYPES: () => MIME_TYPES,
   MissingFieldInSchemaError: () => MissingFieldInSchemaError,
   MissingSchemaError: () => MissingSchemaError,
@@ -10641,7 +10642,7 @@ var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) =>
     ...config.headers ? { headers: config.headers } : {},
     paramsSerializer: { serialize: (params) => qs.stringify(params, { arrayFormat: "repeat" }) }
   }).then((response) => {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolve2, reject) => {
       response.data.pipe(writer);
       let error = null;
       writer.on("error", (err) => {
@@ -10651,7 +10652,7 @@ var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) =>
       });
       writer.on("close", () => {
         if (!error) {
-          resolve({ outputDir: output });
+          resolve2({ outputDir: output });
         }
       });
     });
@@ -10802,8 +10803,8 @@ var printMemoryFootprint = (prefix) => {
 
 // src/sdk/service/exit.ts
 async function gracefulExit(exitCode) {
-  const loggerFinish = new Promise((resolve, reject) => {
-    logger.closeWinston(resolve);
+  const loggerFinish = new Promise((resolve2, reject) => {
+    logger.closeWinston(resolve2);
   });
   await loggerFinish;
   process.exit(exitCode);
@@ -12573,13 +12574,13 @@ var ITarget = class _ITarget {
       const dbSyncInstance = stream.getDbSync();
       const recordWithrenamedColumns = dbSyncInstance.renameColumns(recordWithDecimal);
       const serializedRecord = this.getSerializedRecord(streamId, recordWithrenamedColumns);
-      return new Promise((resolve, reject) => {
+      return new Promise((resolve2, reject) => {
         stream.getFileToLoad().stream.write(serializedRecord, (error) => {
           if (error) {
             reject(error);
           }
           stream.incrementBatchedRowCount();
-          resolve();
+          resolve2();
         });
       });
     } catch (err) {
@@ -12720,11 +12721,11 @@ var ITarget = class _ITarget {
           const streamState = this.streams[streamId];
           if (!streamState || streamState.getBatchedRowCount() === 0) continue;
           const fileToLoad = streamState.getFileToLoad();
-          await new Promise((resolve, reject) => {
+          await new Promise((resolve2, reject) => {
             fileToLoad.stream.end(() => {
               fileToLoad.stream.close((err) => {
                 if (err) reject(err);
-                else resolve();
+                else resolve2();
               });
             });
           });
@@ -13255,18 +13256,18 @@ async function* rowGeneratorFromCsv(path8, fileConfig) {
   }
 }
 async function countCsvLines(filePath) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     let newlines = 0;
     (0, import_fs2.createReadStream)(filePath).on("data", (chunk2) => {
       const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
       for (let i = 0; i < buf.length; i++) {
         if (buf[i] === 10) newlines++;
       }
-    }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
+    }).on("end", () => resolve2(Math.max(0, newlines - 1))).on("error", reject);
   });
 }
 var checkCsvHeaderRow = async (path8, fileConfig) => {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
     console.debug(`${logPrefix} CSV Expected columns:`, colsWithParsingConfig.map((col) => {
       const info = getStringHexInfo(col);
@@ -13341,7 +13342,7 @@ var checkCsvHeaderRow = async (path8, fileConfig) => {
       reject(new Error(`Error processing CSV file: ${error.message}`));
     });
     initialReadStream.on("close", () => {
-      resolve();
+      resolve2();
     });
   });
 };
@@ -13922,6 +13923,16 @@ var CloudStorageService = class {
       throw new Error((0, import_util5.format)(`error writing GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
     }
   }
+  /**
+   * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+   *
+   * @example
+   * const storage = new CloudStorageService("my-bucket", "my-folder");
+   * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+   */
+  async resolveFileUri(fileUri) {
+    return this.downloadFile(fileUri, pathModule.basename(fileUri));
+  }
   /**
    * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
    * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -13950,6 +13961,108 @@ var CloudStorageService = class {
   }
 };
 
+// src/services/local-storage.ts
+var pathModule2 = __toESM(require("path"));
+var fs6 = __toESM(require("fs"));
+var import_crypto2 = require("crypto");
+var logPrefix5 = "[LocalStorageService]";
+var LocalStorageService = class {
+  basePath;
+  processedSuffix;
+  supportedExtensions;
+  constructor(basePath, opts) {
+    this.basePath = pathModule2.resolve(basePath);
+    this.processedSuffix = opts?.processedSuffix ?? ".processed";
+    this.supportedExtensions = opts?.supportedExtensions ?? [];
+  }
+  async listFiles(prefix) {
+    const dir = prefix ? pathModule2.resolve(this.basePath, prefix) : this.basePath;
+    if (!fs6.existsSync(dir)) {
+      return [];
+    }
+    const entries = fs6.readdirSync(dir, { recursive: true });
+    const files = [];
+    for (const entry of entries) {
+      const rel = typeof entry === "string" ? entry : entry.toString();
+      const full = pathModule2.join(dir, rel);
+      if (fs6.statSync(full).isFile()) {
+        files.push(pathModule2.relative(this.basePath, full));
+      }
+    }
+    return files;
+  }
+  async getUnprocessedFiles() {
+    const allFiles = await this.listFiles();
+    const markerFiles = new Set(
+      allFiles.filter((f) => f.endsWith(this.processedSuffix)).map((f) => f.replace(this.processedSuffix, ""))
+    );
+    return allFiles.filter((f) => {
+      if (f.endsWith(this.processedSuffix)) return false;
+      if (this.supportedExtensions.length > 0) {
+        const ext = pathModule2.extname(f).toLowerCase();
+        if (!this.supportedExtensions.includes(ext)) return false;
+      }
+      return !markerFiles.has(f);
+    });
+  }
+  async createMarkerFile(fileName) {
+    const markerPath = pathModule2.join(this.basePath, `${fileName}${this.processedSuffix}`);
+    const dir = pathModule2.dirname(markerPath);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.writeFileSync(markerPath, `Marked file ${fileName} as processed`);
+    logger.info(`${logPrefix5} Marker file ${markerPath} created successfully.`);
+  }
+  async downloadFile(filePath, _fileName) {
+    const resolved = pathModule2.resolve(this.basePath, filePath);
+    if (!fs6.existsSync(resolved)) {
+      throw new Error(`Local file not found: ${resolved}`);
+    }
+    logger.info(`${logPrefix5} Resolved local file: ${resolved}`);
+    return resolved;
+  }
+  async resolveFileUri(fileUri) {
+    const resolved = pathModule2.resolve(fileUri);
+    if (!fs6.existsSync(resolved)) {
+      throw new Error(`Local file not found: ${resolved}`);
+    }
+    logger.info(`${logPrefix5} File URI resolved to local path: ${resolved}`);
+    return resolved;
+  }
+  async uploadFile(localPath, destPath) {
+    const dest = pathModule2.resolve(this.basePath, destPath);
+    const dir = pathModule2.dirname(dest);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.copyFileSync(localPath, dest);
+    logger.info(`${logPrefix5} Copied '%s' to '%s'`, localPath, dest);
+    return { name: destPath };
+  }
+  async readObjectAsString(objectPath) {
+    const fullPath = pathModule2.resolve(this.basePath, objectPath);
+    if (!fs6.existsSync(fullPath)) {
+      throw new Error(`Local file not found: ${fullPath}`);
+    }
+    return fs6.readFileSync(fullPath, "utf8");
+  }
+  async writeStringObject(objectPath, contents) {
+    const fullPath = pathModule2.resolve(this.basePath, objectPath);
+    const dir = pathModule2.dirname(fullPath);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.writeFileSync(fullPath, contents, "utf8");
+    logger.info(`${logPrefix5} Wrote object to ${fullPath}`);
+  }
+  async uploadFileWithUniqueName(filePath, prefix, streamId) {
+    const runFolder = process.env.RUN_ID ?? "default";
+    const destName = `${prefix}/${runFolder}/${streamId}-${(0, import_crypto2.randomUUID)()}.jsonnl`;
+    return this.uploadFile(filePath, destName);
+  }
+};
+
 // src/services/zip.ts
 var fse = __toESM(require("fs-extra"));
 var import_decompress = __toESM(require("decompress"));
@@ -13962,7 +14075,7 @@ var unzip = async (zipFilePath, extractedPath) => {
 // src/services/cdn.ts
 var import_axios3 = __toESM(require("axios"));
 var import_axios_retry2 = __toESM(require("axios-retry"));
-var logPrefix5 = "[CdnService]";
+var logPrefix6 = "[CdnService]";
 var MAX_RETRIES = 10;
 var CdnService = class {
   axiosClient;
@@ -13988,12 +14101,12 @@ var CdnService = class {
         if (retryAfter) {
           const seconds = Number(retryAfter);
           if (!isNaN(seconds) && seconds > 0) {
-            logger.info(`${logPrefix5} Rate limited \u2014 waiting ${seconds}s (Retry-After header), attempt ${retryCount}/${MAX_RETRIES}`);
+            logger.info(`${logPrefix6} Rate limited \u2014 waiting ${seconds}s (Retry-After header), attempt ${retryCount}/${MAX_RETRIES}`);
             return seconds * 1e3;
           }
         }
         const delay = import_axios_retry2.default.exponentialDelay(retryCount);
-        logger.info(`${logPrefix5} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
+        logger.info(`${logPrefix6} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
         return delay;
       }
     });
@@ -14019,7 +14132,7 @@ var CdnService = class {
         const status = err.response?.status;
         if (status === 404) return { exists: false, lastModified: null, contentType: null };
         if (status === 405) {
-          logger.warn(`${logPrefix5} HEAD not supported for ${url}, treating file as absent`);
+          logger.warn(`${logPrefix6} HEAD not supported for ${url}, treating file as absent`);
           return { exists: false, lastModified: null, contentType: null };
         }
         this.throwMeaningfulError(err);
@@ -14043,11 +14156,11 @@ var CdnService = class {
   async uploadFile(cdnId, fileName, fileBuffer) {
     const url = `/v1/cdn/${cdnId}/files/${encodeURIComponent(fileName)}`;
     try {
-      logger.info(`${logPrefix5} Uploading ${fileName} (${fileBuffer.length} bytes) to CDN ${cdnId}`);
+      logger.info(`${logPrefix6} Uploading ${fileName} (${fileBuffer.length} bytes) to CDN ${cdnId}`);
       const form = new FormData();
       form.append("file", new Blob([fileBuffer]), fileName);
       const response = await this.axiosClient.put(url, form);
-      logger.info(`${logPrefix5} Successfully uploaded ${fileName} (status ${response.status})`);
+      logger.info(`${logPrefix6} Successfully uploaded ${fileName} (status ${response.status})`);
       return {
         filePath: response.data?.filePath ?? `/org/${cdnId}/file/${fileName}`
       };
@@ -14090,7 +14203,7 @@ var AssetStream = class {
 // src/sdk/models/asset-tap/asset-tap.ts
 var import_fs_extra4 = __toESM(require("fs-extra"));
 var import_node_path4 = __toESM(require("path"));
-var logPrefix6 = "[AssetTap]";
+var logPrefix7 = "[AssetTap]";
 function inferContentType(filePath, fallback) {
   const mime = getMimeType(filePath);
   return mime !== "application/octet-stream" ? mime : fallback;
@@ -14098,7 +14211,7 @@ function inferContentType(filePath, fallback) {
 function deriveManifestMode(streams) {
   const modes = new Set(streams.map((s) => s.replicationMode));
   if (modes.size > 1) {
-    logger.warn(`${logPrefix6} Streams have mixed replication modes (${[...modes].join(", ")}). Manifest will report "FULL".`);
+    logger.warn(`${logPrefix7} Streams have mixed replication modes (${[...modes].join(", ")}). Manifest will report "FULL".`);
     return "FULL";
   }
   return streams[0]?.replicationMode ?? "INCREMENTAL";
@@ -14120,13 +14233,13 @@ var AssetTap = class {
     const dryRun = isDryRun();
     const dryRunLimit = dryRun ? getDryRunLimit() : void 0;
     if (dryRun) {
-      logger.info(`${logPrefix6} [DRY_RUN] mode active \u2014 skipping CDN checks and uploads`);
+      logger.info(`${logPrefix7} [DRY_RUN] mode active \u2014 skipping CDN checks and uploads`);
       await import_fs_extra4.default.emptyDir(this.outputDir);
       if (dryRunLimit !== void 0) {
-        logger.info(`${logPrefix6} [DRY_RUN] Limit: ${dryRunLimit} assets per stream`);
+        logger.info(`${logPrefix7} [DRY_RUN] Limit: ${dryRunLimit} assets per stream`);
       }
     } else if (process.env["DRY_RUN_LIMIT"] !== void 0) {
-      logger.warn(`${logPrefix6} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
+      logger.warn(`${logPrefix7} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
     }
     const tmpDir2 = import_node_path4.default.join(this.outputDir, "tmp");
     await import_fs_extra4.default.ensureDir(tmpDir2);
@@ -14134,7 +14247,7 @@ var AssetTap = class {
     let entryIndex = 0;
     let totalSummary = { total: 0, uploaded: 0, skipped: 0, errors: 0 };
     for (const stream of this.streams) {
-      logger.info(`${logPrefix6} Processing stream: ${stream.streamId} (mode=${stream.replicationMode}, concurrency=${this.concurrency})`);
+      logger.info(`${logPrefix7} Processing stream: ${stream.streamId} (mode=${stream.replicationMode}, concurrency=${this.concurrency})`);
       const assetEntries = [];
       let streamAssetCount = 0;
       await processFromAsyncIterable(
@@ -14143,11 +14256,11 @@ var AssetTap = class {
           if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
             return "stop";
           }
-          logger.debug(`${logPrefix6} Processing entry: ${entry.sourcePath}`);
+          logger.debug(`${logPrefix7} Processing entry: ${entry.sourcePath}`);
           if (stream.replicationMode === "INCREMENTAL" && !dryRun) {
             const shouldSync = await this.target.shouldSync(entry);
             if (!shouldSync) {
-              logger.debug(`${logPrefix6} Skipping ${entry.sourcePath} (up-to-date)`);
+              logger.debug(`${logPrefix7} Skipping ${entry.sourcePath} (up-to-date)`);
               assetEntries.push({
                 sourcePath: entry.sourcePath,
                 destinationPath: entry.destinationPath,
@@ -14203,10 +14316,10 @@ var AssetTap = class {
               status: "uploaded",
               transformed: wasTransformed
             });
-            logger.info(`${logPrefix6} ${dryRun ? "[DRY_RUN] Processed" : "Uploaded"} ${entry.sourcePath} \u2192 ${entry.destinationPath}`);
+            logger.info(`${logPrefix7} ${dryRun ? "[DRY_RUN] Processed" : "Uploaded"} ${entry.sourcePath} \u2192 ${entry.destinationPath}`);
           } catch (err) {
             const message = err instanceof Error ? err.message : String(err);
-            logger.error(`${logPrefix6} Failed to process ${entry.sourcePath}: ${message}`);
+            logger.error(`${logPrefix7} Failed to process ${entry.sourcePath}: ${message}`);
             assetEntries.push({
               sourcePath: entry.sourcePath,
               destinationPath: entry.destinationPath,
@@ -14225,7 +14338,7 @@ var AssetTap = class {
             }
           }
           if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
-            logger.info(`${logPrefix6} [DRY_RUN] Reached limit of ${dryRunLimit} for stream "${stream.streamId}", stopping.`);
+            logger.info(`${logPrefix7} [DRY_RUN] Reached limit of ${dryRunLimit} for stream "${stream.streamId}", stopping.`);
             return "stop";
           }
           return "continue";
@@ -14261,7 +14374,7 @@ var AssetTap = class {
     if (!dryRun) {
       await this.target.complete();
     }
-    logger.info(`${logPrefix6} Sync complete. Uploaded=${totalSummary.uploaded} Skipped=${totalSummary.skipped} Errors=${totalSummary.errors}`);
+    logger.info(`${logPrefix7} Sync complete. Uploaded=${totalSummary.uploaded} Skipped=${totalSummary.skipped} Errors=${totalSummary.errors}`);
     return manifest;
   }
 };
@@ -14299,8 +14412,8 @@ var ImageTransform = class {
    */
   static async toWebp(inputPath, options) {
     const dir = import_node_path5.default.dirname(inputPath);
-    const basename = import_node_path5.default.basename(inputPath, import_node_path5.default.extname(inputPath));
-    const outputPath = import_node_path5.default.join(dir, `${basename}.webp`);
+    const basename2 = import_node_path5.default.basename(inputPath, import_node_path5.default.extname(inputPath));
+    const outputPath = import_node_path5.default.join(dir, `${basename2}.webp`);
     let pipeline = (0, import_sharp.default)(inputPath);
     const hasSize = options.width !== void 0 && options.height !== void 0;
     if (hasSize) {
@@ -14424,7 +14537,7 @@ async function getFileSizeLabel(filePath) {
     return "unknown size";
   }
 }
-var logPrefix7 = "[DocumentTap]";
+var logPrefix8 = "[DocumentTap]";
 var DocumentTap = class {
   config;
   outputDir;
@@ -14443,32 +14556,32 @@ var DocumentTap = class {
   }
   async sync() {
     if (!this.target) {
-      throw new Error(`${logPrefix7} No target set. Assign a WhalyDocumentTarget before calling sync().`);
+      throw new Error(`${logPrefix8} No target set. Assign a WhalyDocumentTarget before calling sync().`);
     }
     await this.init();
     if (!this.stream) {
-      throw new Error(`${logPrefix7} No stream set. Assign this.stream in init().`);
+      throw new Error(`${logPrefix8} No stream set. Assign this.stream in init().`);
     }
     const dryRun = isDryRun();
     const dryRunLimit = dryRun ? getDryRunLimit() : void 0;
     if (dryRun) {
-      logger.info(`${logPrefix7} [DRY_RUN] mode active \u2014 skipping API calls and uploads`);
+      logger.info(`${logPrefix8} [DRY_RUN] mode active \u2014 skipping API calls and uploads`);
       await import_fs_extra5.default.emptyDir(this.outputDir);
       if (dryRunLimit !== void 0) {
-        logger.info(`${logPrefix7} [DRY_RUN] Limit: ${dryRunLimit} documents per stream`);
+        logger.info(`${logPrefix8} [DRY_RUN] Limit: ${dryRunLimit} documents per stream`);
       }
     } else if (process.env["DRY_RUN_LIMIT"] !== void 0) {
-      logger.warn(`${logPrefix7} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
+      logger.warn(`${logPrefix8} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
     }
     const tmpDir2 = import_node_path6.default.join(this.outputDir, "tmp");
     await import_fs_extra5.default.ensureDir(tmpDir2);
     try {
       const stream = this.stream;
-      logger.info(`${logPrefix7} Processing stream: ${stream.streamId}`);
+      logger.info(`${logPrefix8} Processing stream: ${stream.streamId}`);
       const sourceEntries = await this.collectSourceEntries(stream, dryRunLimit);
       const existingDocs = dryRun ? [] : await this.target.listExistingDocuments();
       const diff = this.computeDiff(stream, sourceEntries, existingDocs);
-      logger.info(`${logPrefix7} Stream ${stream.streamId} diff: create=${diff.toCreate.length} reupload=${diff.toReupload.length} updateMeta=${diff.toUpdateMetadata.length} delete=${diff.toDelete.length} skip=${diff.skipped.length}`);
+      logger.info(`${logPrefix8} Stream ${stream.streamId} diff: create=${diff.toCreate.length} reupload=${diff.toReupload.length} updateMeta=${diff.toUpdateMetadata.length} delete=${diff.toDelete.length} skip=${diff.skipped.length}`);
       const entries = [];
       const createTasks = diff.toCreate.map((entry) => async () => {
         return this.executeCreate(stream, entry, tmpDir2, dryRun);
@@ -14502,7 +14615,7 @@ var DocumentTap = class {
       };
       await import_fs_extra5.default.ensureDir(this.outputDir);
       await import_fs_extra5.default.writeJson(import_node_path6.default.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
-      logger.info(`${logPrefix7} Sync complete. Created=${streamSummary.created} Updated=${streamSummary.updated} Reuploaded=${streamSummary.reuploaded} Deleted=${streamSummary.deleted} Skipped=${streamSummary.skipped} Errors=${streamSummary.errors}`);
+      logger.info(`${logPrefix8} Sync complete. Created=${streamSummary.created} Updated=${streamSummary.updated} Reuploaded=${streamSummary.reuploaded} Deleted=${streamSummary.deleted} Skipped=${streamSummary.skipped} Errors=${streamSummary.errors}`);
       return manifest;
     } finally {
       await import_fs_extra5.default.remove(tmpDir2).catch(() => void 0);
@@ -14570,12 +14683,12 @@ var DocumentTap = class {
       return { externalId: entry.externalId, fileName: entry.fileName, status: "created" };
     } catch (err) {
       if (err instanceof DocumentDownloadSkipError) {
-        logger.warn(`${logPrefix7} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
+        logger.warn(`${logPrefix8} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
         return { externalId: entry.externalId, fileName: entry.fileName, status: "skipped" };
       }
       const message = err instanceof Error ? err.message : String(err);
       const sizeLabel = await getFileSizeLabel(downloadPath);
-      logger.error(`${logPrefix7} Failed to create ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
+      logger.error(`${logPrefix8} Failed to create ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     } finally {
       await import_fs_extra5.default.remove(downloadPath).catch(() => void 0);
@@ -14596,12 +14709,12 @@ var DocumentTap = class {
       return { externalId: entry.externalId, fileName: entry.fileName, status: "reuploaded" };
     } catch (err) {
       if (err instanceof DocumentDownloadSkipError) {
-        logger.warn(`${logPrefix7} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
+        logger.warn(`${logPrefix8} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
         return { externalId: entry.externalId, fileName: entry.fileName, status: "skipped" };
       }
       const message = err instanceof Error ? err.message : String(err);
       const sizeLabel = await getFileSizeLabel(downloadPath);
-      logger.error(`${logPrefix7} Failed to reupload ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
+      logger.error(`${logPrefix8} Failed to reupload ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     } finally {
       await import_fs_extra5.default.remove(downloadPath).catch(() => void 0);
@@ -14615,7 +14728,7 @@ var DocumentTap = class {
       return { externalId: entry.externalId, fileName: entry.fileName, status: "updated" };
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      logger.error(`${logPrefix7} Failed to update metadata for ${entry.externalId}: ${message}`);
+      logger.error(`${logPrefix8} Failed to update metadata for ${entry.externalId}: ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     }
   }
@@ -14627,7 +14740,7 @@ var DocumentTap = class {
       return { externalId: doc.external_id, fileName: doc.file_name, status: "deleted" };
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      logger.error(`${logPrefix7} Failed to delete ${doc.external_id}: ${message}`);
+      logger.error(`${logPrefix8} Failed to delete ${doc.external_id}: ${message}`);
       return { externalId: doc.external_id, fileName: doc.file_name, status: "error", error: message };
     }
   }
@@ -14664,20 +14777,22 @@ function enrichAxiosError(err) {
   const detail = typeof body === "string" ? body : JSON.stringify(body ?? "");
   return new Error(`${method} ${url} failed with status ${status}: ${detail}`, { cause: err });
 }
-var logPrefix8 = "[WhalyDocumentService]";
+var logPrefix9 = "[WhalyDocumentService]";
 var MAX_RETRIES2 = 10;
 var WhalyDocumentService = class {
   axiosClient;
   objectStorageId;
+  documentSourceId;
   gcsBucket;
   constructor(config) {
     const resolvedKey = getServiceAccountKey(config.serviceAccountKey);
     const resolvedEndpoint = getApiEndpoint(config.apiEndpoint);
     this.objectStorageId = config.objectStorageId;
+    this.documentSourceId = config.documentSourceId;
     const gcsBucketName = process.env["WLY_GCS_BUCKET"];
     if (gcsBucketName) {
       this.gcsBucket = new import_storage2.Storage().bucket(gcsBucketName);
-      logger.info(`${logPrefix8} Using direct GCS upload to bucket: ${gcsBucketName}`);
+      logger.info(`${logPrefix9} Using direct GCS upload to bucket: ${gcsBucketName}`);
     }
     this.axiosClient = import_axios4.default.create({
       baseURL: resolvedEndpoint,
@@ -14698,22 +14813,26 @@ var WhalyDocumentService = class {
         if (retryAfter) {
           const seconds = Number(retryAfter);
           if (!isNaN(seconds) && seconds > 0) {
-            logger.info(`${logPrefix8} Rate limited \u2014 waiting ${seconds}s, attempt ${retryCount}/${MAX_RETRIES2}`);
+            logger.info(`${logPrefix9} Rate limited \u2014 waiting ${seconds}s, attempt ${retryCount}/${MAX_RETRIES2}`);
             return seconds * 1e3;
           }
         }
         const delay = import_axios_retry3.default.exponentialDelay(retryCount);
-        logger.info(`${logPrefix8} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES2}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
+        logger.info(`${logPrefix9} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES2}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
         return delay;
       }
     });
   }
-  /** Fetch all documents from the API, handling cursor-based pagination. */
+  /**
+   * Fetch all documents belonging to the configured document source,
+   * handling cursor-based pagination. The `document_source_id` server-side
+   * filter scopes the result to this connector's source only.
+   */
   async listAllDocuments() {
     const documents = [];
     let after;
     while (true) {
-      const params = {};
+      const params = { document_source_id: this.documentSourceId };
       if (after) params["after"] = after;
       const response = await this.axiosClient.get(
         "/v1/documents",
@@ -14724,7 +14843,7 @@ var WhalyDocumentService = class {
       if (!nextAfter) break;
       after = nextAfter;
     }
-    logger.info(`${logPrefix8} Fetched ${documents.length} existing documents from Whaly`);
+    logger.info(`${logPrefix9} Fetched ${documents.length} existing documents from Whaly`);
     return documents;
   }
   /** Upload a file to object storage. Returns storage path and size. */
@@ -14745,8 +14864,8 @@ var WhalyDocumentService = class {
       } catch (err) {
         if (attempt === maxAttempts) throw err;
         const delay = Math.min(1e3 * 2 ** (attempt - 1), 3e4);
-        logger.info(`${logPrefix8} GCS upload retry ${attempt}/${MAX_RETRIES2} in ${delay}ms for ${destinationPath}`);
-        await new Promise((resolve) => setTimeout(resolve, delay));
+        logger.info(`${logPrefix9} GCS upload retry ${attempt}/${MAX_RETRIES2} in ${delay}ms for ${destinationPath}`);
+        await new Promise((resolve2) => setTimeout(resolve2, delay));
       }
     }
     const stat = await import_node_fs.default.promises.stat(localFilePath);
@@ -14773,19 +14892,26 @@ var WhalyDocumentService = class {
       throw enrichAxiosError(err);
     }
   }
-  /** Create a new document record. */
+  /** Create a new document record, attached to the configured document source. */
   async createDocument(payload) {
     try {
-      const response = await this.axiosClient.post("/v1/documents", payload);
+      const response = await this.axiosClient.post("/v1/documents", {
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
     }
   }
-  /** Update an existing document record. */
+  /** Update an existing document record, keeping it in the configured document source. */
   async updateDocument(id, payload) {
     try {
-      const response = await this.axiosClient.put(`/v1/documents/${id}`, { id, ...payload });
+      const response = await this.axiosClient.put(`/v1/documents/${id}`, {
+        id,
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
@@ -14802,7 +14928,7 @@ var WhalyDocumentService = class {
 };
 
 // src/sdk/models/document-target/whaly-document-target.ts
-var logPrefix9 = "[WhalyDocumentTarget]";
+var logPrefix10 = "[WhalyDocumentTarget]";
 var WhalyDocumentTarget = class {
   config;
   service;
@@ -14821,7 +14947,7 @@ var WhalyDocumentTarget = class {
   async createDocument(streamId, entry, localFilePath) {
     const destinationPath = `${streamId}/${entry.externalId}.${entry.extension}`;
     const fileName = `${entry.externalId}.${entry.extension}`;
-    logger.info(`${logPrefix9} Uploading file for new document: ${entry.externalId}`);
+    logger.info(`${logPrefix10} Uploading file for new document: ${entry.externalId}`);
     const uploadResult = await this.service.uploadFile(destinationPath, localFilePath, fileName);
     const stat = await import_fs_extra6.default.stat(localFilePath);
     await this.service.createDocument({
@@ -14838,7 +14964,7 @@ var WhalyDocumentTarget = class {
       storage: uploadResult.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Created document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Created document: ${entry.externalId} (${entry.fileName})`);
   }
   /**
    * Re-upload the file and update the document record.
@@ -14846,7 +14972,7 @@ var WhalyDocumentTarget = class {
   async reuploadDocument(streamId, docId, entry, localFilePath) {
     const destinationPath = `${streamId}/${entry.externalId}.${entry.extension}`;
     const fileName = `${entry.externalId}.${entry.extension}`;
-    logger.info(`${logPrefix9} Re-uploading file for document: ${entry.externalId}`);
+    logger.info(`${logPrefix10} Re-uploading file for document: ${entry.externalId}`);
     const uploadResult = await this.service.uploadFile(destinationPath, localFilePath, fileName);
     const stat = await import_fs_extra6.default.stat(localFilePath);
     await this.service.updateDocument(docId, {
@@ -14863,7 +14989,7 @@ var WhalyDocumentTarget = class {
       storage: uploadResult.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Re-uploaded document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Re-uploaded document: ${entry.externalId} (${entry.fileName})`);
   }
   /** Update only the metadata fields (no file re-upload). */
   async updateDocumentMetadata(docId, entry, existingDoc) {
@@ -14881,12 +15007,12 @@ var WhalyDocumentTarget = class {
       storage: existingDoc.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Updated metadata for document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Updated metadata for document: ${entry.externalId} (${entry.fileName})`);
   }
   /** Delete a document record. */
   async deleteDocument(docId, externalId) {
     await this.service.deleteDocument(docId);
-    logger.info(`${logPrefix9} Deleted document: ${externalId} (id=${docId})`);
+    logger.info(`${logPrefix10} Deleted document: ${externalId} (id=${docId})`);
   }
 };
 
@@ -15225,7 +15351,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
         gracefulExit(1);
       } else {
         logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. Retrying... ${err.message} - ${err.code}`);
-        await new Promise((resolve, reject) => setTimeout(resolve, 1e3 * retryCount));
+        await new Promise((resolve2, reject) => setTimeout(resolve2, 1e3 * retryCount));
         await this.createDatabaseAndSchemaIfNotExists(retryCount + 1);
       }
     }
@@ -15453,7 +15579,7 @@ var BigQueryTarget = class extends ITarget {
 
 // src/targets/cdn/main.ts
 var import_fs_extra7 = __toESM(require("fs-extra"));
-var logPrefix10 = "[CdnAssetTarget]";
+var logPrefix11 = "[CdnAssetTarget]";
 var CdnAssetTarget = class extends AssetTarget {
   cdnService;
   constructor(config) {
@@ -15463,20 +15589,20 @@ var CdnAssetTarget = class extends AssetTarget {
   async shouldSync(entry) {
     const metadata = await this.cdnService.getFileMetadata(getCdnId(this.config.cdnId), entry.destinationPath);
     if (!metadata.exists) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} not in CDN \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} not in CDN \u2192 will sync`);
       return true;
     }
     if (entry.lastModified === void 0) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} source has no lastModified \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} source has no lastModified \u2192 will sync`);
       return true;
     }
     if (metadata.lastModified === null) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} CDN has no lastModified \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} CDN has no lastModified \u2192 will sync`);
       return true;
     }
     const sourceIsNewer = entry.lastModified > metadata.lastModified;
     if (!sourceIsNewer) {
-      logger.debug(`${logPrefix10} Skipping ${entry.destinationPath} (CDN is up-to-date)`);
+      logger.debug(`${logPrefix11} Skipping ${entry.destinationPath} (CDN is up-to-date)`);
     }
     return sourceIsNewer;
   }
@@ -15487,7 +15613,7 @@ var CdnAssetTarget = class extends AssetTarget {
       asset.entry.destinationPath,
       fileBuffer
     );
-    logger.info(`${logPrefix10} Uploaded ${asset.entry.destinationPath} (${fileBuffer.length} bytes)`);
+    logger.info(`${logPrefix11} Uploaded ${asset.entry.destinationPath} (${fileBuffer.length} bytes)`);
   }
 };
 
@@ -15557,6 +15683,7 @@ var GCSStateProvider = class {
   GCSStateProvider,
   ITarget,
   ImageTransform,
+  LocalStorageService,
   MIME_TYPES,
   MissingFieldInSchemaError,
   MissingSchemaError,