@whaly/connector-sdk 0.3.10 → 0.3.12

package/dist/index.mjs

@@ -8830,7 +8830,7 @@ var require_mime_types = __commonJS({
   "node_modules/mime-types/index.js"(exports) {
     "use strict";
     var db = require_mime_db();
-    var extname2 = __require("path").extname;
+    var extname3 = __require("path").extname;
     var EXTRACT_TYPE_REGEXP = /^\s*([^;\s]*)(?:;|\s|$)/;
     var TEXT_TYPE_REGEXP = /^text\//i;
     exports.charset = charset;
@@ -8884,7 +8884,7 @@ var require_mime_types = __commonJS({
       if (!path8 || typeof path8 !== "string") {
         return false;
       }
-      var extension2 = extname2("x." + path8).toLowerCase().substr(1);
+      var extension2 = extname3("x." + path8).toLowerCase().substr(1);
       if (!extension2) {
         return false;
       }
@@ -10003,7 +10003,7 @@ var require_form_data = __commonJS({
     var http = __require("http");
     var https = __require("https");
     var parseUrl = __require("url").parse;
-    var fs11 = __require("fs");
+    var fs12 = __require("fs");
     var Stream2 = __require("stream").Stream;
     var crypto = __require("crypto");
     var mime = require_mime_types();
@@ -10070,7 +10070,7 @@ var require_form_data = __commonJS({
         if (value.end != void 0 && value.end != Infinity && value.start != void 0) {
           callback(null, value.end + 1 - (value.start ? value.start : 0));
         } else {
-          fs11.stat(value.path, function(err, stat) {
+          fs12.stat(value.path, function(err, stat) {
             if (err) {
               callback(err);
               return;
@@ -10520,7 +10520,7 @@ var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) =>
     ...config.headers ? { headers: config.headers } : {},
     paramsSerializer: { serialize: (params) => qs.stringify(params, { arrayFormat: "repeat" }) }
   }).then((response) => {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolve2, reject) => {
       response.data.pipe(writer);
       let error = null;
       writer.on("error", (err) => {
@@ -10530,7 +10530,7 @@ var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) =>
       });
       writer.on("close", () => {
         if (!error) {
-          resolve({ outputDir: output });
+          resolve2({ outputDir: output });
         }
       });
     });
@@ -10681,8 +10681,8 @@ var printMemoryFootprint = (prefix) => {
 
 // src/sdk/service/exit.ts
 async function gracefulExit(exitCode) {
-  const loggerFinish = new Promise((resolve, reject) => {
-    logger.closeWinston(resolve);
+  const loggerFinish = new Promise((resolve2, reject) => {
+    logger.closeWinston(resolve2);
   });
   await loggerFinish;
   process.exit(exitCode);
@@ -12452,13 +12452,13 @@ var ITarget = class _ITarget {
       const dbSyncInstance = stream.getDbSync();
       const recordWithrenamedColumns = dbSyncInstance.renameColumns(recordWithDecimal);
       const serializedRecord = this.getSerializedRecord(streamId, recordWithrenamedColumns);
-      return new Promise((resolve, reject) => {
+      return new Promise((resolve2, reject) => {
         stream.getFileToLoad().stream.write(serializedRecord, (error) => {
           if (error) {
             reject(error);
           }
           stream.incrementBatchedRowCount();
-          resolve();
+          resolve2();
         });
       });
     } catch (err) {
@@ -12599,11 +12599,11 @@ var ITarget = class _ITarget {
           const streamState = this.streams[streamId];
           if (!streamState || streamState.getBatchedRowCount() === 0) continue;
           const fileToLoad = streamState.getFileToLoad();
-          await new Promise((resolve, reject) => {
+          await new Promise((resolve2, reject) => {
             fileToLoad.stream.end(() => {
               fileToLoad.stream.close((err) => {
                 if (err) reject(err);
-                else resolve();
+                else resolve2();
               });
             });
           });
@@ -13134,18 +13134,18 @@ async function* rowGeneratorFromCsv(path8, fileConfig) {
   }
 }
 async function countCsvLines(filePath) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     let newlines = 0;
     createReadStream(filePath).on("data", (chunk2) => {
       const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
       for (let i = 0; i < buf.length; i++) {
         if (buf[i] === 10) newlines++;
       }
-    }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
+    }).on("end", () => resolve2(Math.max(0, newlines - 1))).on("error", reject);
   });
 }
 var checkCsvHeaderRow = async (path8, fileConfig) => {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
     console.debug(`${logPrefix} CSV Expected columns:`, colsWithParsingConfig.map((col) => {
       const info = getStringHexInfo(col);
@@ -13220,7 +13220,7 @@ var checkCsvHeaderRow = async (path8, fileConfig) => {
       reject(new Error(`Error processing CSV file: ${error.message}`));
     });
     initialReadStream.on("close", () => {
-      resolve();
+      resolve2();
     });
   });
 };
@@ -13801,6 +13801,16 @@ var CloudStorageService = class {
       throw new Error(format6(`error writing GCS object gs://${this.bucket.name}/${objectPath}, err: %s`, err?.message));
     }
   }
+  /**
+   * Resolves a file URI to a local file path by downloading it from the configured GCS bucket.
+   *
+   * @example
+   * const storage = new CloudStorageService("my-bucket", "my-folder");
+   * const localPath = await storage.resolveFileUri("folder/file.xlsx");
+   */
+  async resolveFileUri(fileUri) {
+    return this.downloadFile(fileUri, pathModule.basename(fileUri));
+  }
   /**
    * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
    * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
@@ -13829,6 +13839,108 @@ var CloudStorageService = class {
   }
 };
 
+// src/services/local-storage.ts
+import * as pathModule2 from "path";
+import * as fs6 from "fs";
+import { randomUUID as randomUUID2 } from "crypto";
+var logPrefix5 = "[LocalStorageService]";
+var LocalStorageService = class {
+  basePath;
+  processedSuffix;
+  supportedExtensions;
+  constructor(basePath, opts) {
+    this.basePath = pathModule2.resolve(basePath);
+    this.processedSuffix = opts?.processedSuffix ?? ".processed";
+    this.supportedExtensions = opts?.supportedExtensions ?? [];
+  }
+  async listFiles(prefix) {
+    const dir = prefix ? pathModule2.resolve(this.basePath, prefix) : this.basePath;
+    if (!fs6.existsSync(dir)) {
+      return [];
+    }
+    const entries = fs6.readdirSync(dir, { recursive: true });
+    const files = [];
+    for (const entry of entries) {
+      const rel = typeof entry === "string" ? entry : entry.toString();
+      const full = pathModule2.join(dir, rel);
+      if (fs6.statSync(full).isFile()) {
+        files.push(pathModule2.relative(this.basePath, full));
+      }
+    }
+    return files;
+  }
+  async getUnprocessedFiles() {
+    const allFiles = await this.listFiles();
+    const markerFiles = new Set(
+      allFiles.filter((f) => f.endsWith(this.processedSuffix)).map((f) => f.replace(this.processedSuffix, ""))
+    );
+    return allFiles.filter((f) => {
+      if (f.endsWith(this.processedSuffix)) return false;
+      if (this.supportedExtensions.length > 0) {
+        const ext = pathModule2.extname(f).toLowerCase();
+        if (!this.supportedExtensions.includes(ext)) return false;
+      }
+      return !markerFiles.has(f);
+    });
+  }
+  async createMarkerFile(fileName) {
+    const markerPath = pathModule2.join(this.basePath, `${fileName}${this.processedSuffix}`);
+    const dir = pathModule2.dirname(markerPath);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.writeFileSync(markerPath, `Marked file ${fileName} as processed`);
+    logger.info(`${logPrefix5} Marker file ${markerPath} created successfully.`);
+  }
+  async downloadFile(filePath, _fileName) {
+    const resolved = pathModule2.resolve(this.basePath, filePath);
+    if (!fs6.existsSync(resolved)) {
+      throw new Error(`Local file not found: ${resolved}`);
+    }
+    logger.info(`${logPrefix5} Resolved local file: ${resolved}`);
+    return resolved;
+  }
+  async resolveFileUri(fileUri) {
+    const resolved = pathModule2.resolve(fileUri);
+    if (!fs6.existsSync(resolved)) {
+      throw new Error(`Local file not found: ${resolved}`);
+    }
+    logger.info(`${logPrefix5} File URI resolved to local path: ${resolved}`);
+    return resolved;
+  }
+  async uploadFile(localPath, destPath) {
+    const dest = pathModule2.resolve(this.basePath, destPath);
+    const dir = pathModule2.dirname(dest);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.copyFileSync(localPath, dest);
+    logger.info(`${logPrefix5} Copied '%s' to '%s'`, localPath, dest);
+    return { name: destPath };
+  }
+  async readObjectAsString(objectPath) {
+    const fullPath = pathModule2.resolve(this.basePath, objectPath);
+    if (!fs6.existsSync(fullPath)) {
+      throw new Error(`Local file not found: ${fullPath}`);
+    }
+    return fs6.readFileSync(fullPath, "utf8");
+  }
+  async writeStringObject(objectPath, contents) {
+    const fullPath = pathModule2.resolve(this.basePath, objectPath);
+    const dir = pathModule2.dirname(fullPath);
+    if (!fs6.existsSync(dir)) {
+      fs6.mkdirSync(dir, { recursive: true });
+    }
+    fs6.writeFileSync(fullPath, contents, "utf8");
+    logger.info(`${logPrefix5} Wrote object to ${fullPath}`);
+  }
+  async uploadFileWithUniqueName(filePath, prefix, streamId) {
+    const runFolder = process.env.RUN_ID ?? "default";
+    const destName = `${prefix}/${runFolder}/${streamId}-${randomUUID2()}.jsonnl`;
+    return this.uploadFile(filePath, destName);
+  }
+};
+
 // src/services/zip.ts
 import * as fse from "fs-extra";
 import decompress from "decompress";
@@ -13841,7 +13953,7 @@ var unzip = async (zipFilePath, extractedPath) => {
 // src/services/cdn.ts
 import axios3 from "axios";
 import axiosRetry2 from "axios-retry";
-var logPrefix5 = "[CdnService]";
+var logPrefix6 = "[CdnService]";
 var MAX_RETRIES = 10;
 var CdnService = class {
   axiosClient;
@@ -13867,12 +13979,12 @@ var CdnService = class {
         if (retryAfter) {
           const seconds = Number(retryAfter);
           if (!isNaN(seconds) && seconds > 0) {
-            logger.info(`${logPrefix5} Rate limited \u2014 waiting ${seconds}s (Retry-After header), attempt ${retryCount}/${MAX_RETRIES}`);
+            logger.info(`${logPrefix6} Rate limited \u2014 waiting ${seconds}s (Retry-After header), attempt ${retryCount}/${MAX_RETRIES}`);
             return seconds * 1e3;
           }
         }
         const delay = axiosRetry2.exponentialDelay(retryCount);
-        logger.info(`${logPrefix5} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
+        logger.info(`${logPrefix6} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
         return delay;
       }
     });
@@ -13898,7 +14010,7 @@ var CdnService = class {
         const status = err.response?.status;
         if (status === 404) return { exists: false, lastModified: null, contentType: null };
         if (status === 405) {
-          logger.warn(`${logPrefix5} HEAD not supported for ${url}, treating file as absent`);
+          logger.warn(`${logPrefix6} HEAD not supported for ${url}, treating file as absent`);
           return { exists: false, lastModified: null, contentType: null };
         }
         this.throwMeaningfulError(err);
@@ -13922,11 +14034,11 @@ var CdnService = class {
   async uploadFile(cdnId, fileName, fileBuffer) {
     const url = `/v1/cdn/${cdnId}/files/${encodeURIComponent(fileName)}`;
     try {
-      logger.info(`${logPrefix5} Uploading ${fileName} (${fileBuffer.length} bytes) to CDN ${cdnId}`);
+      logger.info(`${logPrefix6} Uploading ${fileName} (${fileBuffer.length} bytes) to CDN ${cdnId}`);
       const form = new FormData();
       form.append("file", new Blob([fileBuffer]), fileName);
       const response = await this.axiosClient.put(url, form);
-      logger.info(`${logPrefix5} Successfully uploaded ${fileName} (status ${response.status})`);
+      logger.info(`${logPrefix6} Successfully uploaded ${fileName} (status ${response.status})`);
       return {
         filePath: response.data?.filePath ?? `/org/${cdnId}/file/${fileName}`
       };
@@ -13967,9 +14079,9 @@ var AssetStream = class {
 };
 
 // src/sdk/models/asset-tap/asset-tap.ts
-import fs6 from "fs-extra";
+import fs7 from "fs-extra";
 import path5 from "path";
-var logPrefix6 = "[AssetTap]";
+var logPrefix7 = "[AssetTap]";
 function inferContentType(filePath, fallback) {
   const mime = getMimeType(filePath);
   return mime !== "application/octet-stream" ? mime : fallback;
@@ -13977,7 +14089,7 @@ function inferContentType(filePath, fallback) {
 function deriveManifestMode(streams) {
   const modes = new Set(streams.map((s) => s.replicationMode));
   if (modes.size > 1) {
-    logger.warn(`${logPrefix6} Streams have mixed replication modes (${[...modes].join(", ")}). Manifest will report "FULL".`);
+    logger.warn(`${logPrefix7} Streams have mixed replication modes (${[...modes].join(", ")}). Manifest will report "FULL".`);
     return "FULL";
   }
   return streams[0]?.replicationMode ?? "INCREMENTAL";
@@ -13999,21 +14111,21 @@ var AssetTap = class {
     const dryRun = isDryRun();
     const dryRunLimit = dryRun ? getDryRunLimit() : void 0;
     if (dryRun) {
-      logger.info(`${logPrefix6} [DRY_RUN] mode active \u2014 skipping CDN checks and uploads`);
-      await fs6.emptyDir(this.outputDir);
+      logger.info(`${logPrefix7} [DRY_RUN] mode active \u2014 skipping CDN checks and uploads`);
+      await fs7.emptyDir(this.outputDir);
       if (dryRunLimit !== void 0) {
-        logger.info(`${logPrefix6} [DRY_RUN] Limit: ${dryRunLimit} assets per stream`);
+        logger.info(`${logPrefix7} [DRY_RUN] Limit: ${dryRunLimit} assets per stream`);
       }
     } else if (process.env["DRY_RUN_LIMIT"] !== void 0) {
-      logger.warn(`${logPrefix6} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
+      logger.warn(`${logPrefix7} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
     }
     const tmpDir2 = path5.join(this.outputDir, "tmp");
-    await fs6.ensureDir(tmpDir2);
+    await fs7.ensureDir(tmpDir2);
     const streamManifests = [];
     let entryIndex = 0;
     let totalSummary = { total: 0, uploaded: 0, skipped: 0, errors: 0 };
     for (const stream of this.streams) {
-      logger.info(`${logPrefix6} Processing stream: ${stream.streamId} (mode=${stream.replicationMode}, concurrency=${this.concurrency})`);
+      logger.info(`${logPrefix7} Processing stream: ${stream.streamId} (mode=${stream.replicationMode}, concurrency=${this.concurrency})`);
       const assetEntries = [];
       let streamAssetCount = 0;
       await processFromAsyncIterable(
@@ -14022,11 +14134,11 @@ var AssetTap = class {
           if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
             return "stop";
           }
-          logger.debug(`${logPrefix6} Processing entry: ${entry.sourcePath}`);
+          logger.debug(`${logPrefix7} Processing entry: ${entry.sourcePath}`);
           if (stream.replicationMode === "INCREMENTAL" && !dryRun) {
             const shouldSync = await this.target.shouldSync(entry);
             if (!shouldSync) {
-              logger.debug(`${logPrefix6} Skipping ${entry.sourcePath} (up-to-date)`);
+              logger.debug(`${logPrefix7} Skipping ${entry.sourcePath} (up-to-date)`);
               assetEntries.push({
                 sourcePath: entry.sourcePath,
                 destinationPath: entry.destinationPath,
@@ -14049,7 +14161,7 @@ var AssetTap = class {
             await stream.downloadEntry(entry, downloadedPath);
             uploadPath = await stream.transformFile(downloadedPath, entry);
             const wasTransformed = uploadPath !== downloadedPath;
-            const stat = await fs6.stat(uploadPath);
+            const stat = await fs7.stat(uploadPath);
             const processed = {
               entry,
               downloadedPath,
@@ -14062,14 +14174,14 @@ var AssetTap = class {
               await this.target.uploadAsset(processed);
             } else {
               const inspectPath = path5.join(this.outputDir, stream.streamId, entry.destinationPath);
-              await fs6.ensureDir(path5.dirname(inspectPath));
-              await fs6.copy(uploadPath, inspectPath);
+              await fs7.ensureDir(path5.dirname(inspectPath));
+              await fs7.copy(uploadPath, inspectPath);
               if (wasTransformed) {
                 const baseName = path5.basename(entry.destinationPath, path5.extname(entry.destinationPath));
                 const origExt = getExtension(entry.contentType);
                 const originalPath = path5.join(this.outputDir, "originals", `${baseName}${origExt}`);
-                await fs6.ensureDir(path5.dirname(originalPath));
-                await fs6.copy(downloadedPath, originalPath);
+                await fs7.ensureDir(path5.dirname(originalPath));
+                await fs7.copy(downloadedPath, originalPath);
               }
             }
             assetEntries.push({
@@ -14082,10 +14194,10 @@ var AssetTap = class {
               status: "uploaded",
               transformed: wasTransformed
             });
-            logger.info(`${logPrefix6} ${dryRun ? "[DRY_RUN] Processed" : "Uploaded"} ${entry.sourcePath} \u2192 ${entry.destinationPath}`);
+            logger.info(`${logPrefix7} ${dryRun ? "[DRY_RUN] Processed" : "Uploaded"} ${entry.sourcePath} \u2192 ${entry.destinationPath}`);
           } catch (err) {
             const message = err instanceof Error ? err.message : String(err);
-            logger.error(`${logPrefix6} Failed to process ${entry.sourcePath}: ${message}`);
+            logger.error(`${logPrefix7} Failed to process ${entry.sourcePath}: ${message}`);
             assetEntries.push({
               sourcePath: entry.sourcePath,
               destinationPath: entry.destinationPath,
@@ -14098,13 +14210,13 @@ var AssetTap = class {
               error: message
             });
           } finally {
-            await fs6.remove(downloadedPath).catch(() => void 0);
+            await fs7.remove(downloadedPath).catch(() => void 0);
             if (uploadPath !== downloadedPath) {
-              await fs6.remove(uploadPath).catch(() => void 0);
+              await fs7.remove(uploadPath).catch(() => void 0);
             }
           }
           if (dryRunLimit !== void 0 && streamAssetCount >= dryRunLimit) {
-            logger.info(`${logPrefix6} [DRY_RUN] Reached limit of ${dryRunLimit} for stream "${stream.streamId}", stopping.`);
+            logger.info(`${logPrefix7} [DRY_RUN] Reached limit of ${dryRunLimit} for stream "${stream.streamId}", stopping.`);
             return "stop";
           }
           return "continue";
@@ -14135,12 +14247,12 @@ var AssetTap = class {
       streams: streamManifests,
       summary: totalSummary
     };
-    await fs6.ensureDir(this.outputDir);
-    await fs6.writeJson(path5.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
+    await fs7.ensureDir(this.outputDir);
+    await fs7.writeJson(path5.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
     if (!dryRun) {
       await this.target.complete();
     }
-    logger.info(`${logPrefix6} Sync complete. Uploaded=${totalSummary.uploaded} Skipped=${totalSummary.skipped} Errors=${totalSummary.errors}`);
+    logger.info(`${logPrefix7} Sync complete. Uploaded=${totalSummary.uploaded} Skipped=${totalSummary.skipped} Errors=${totalSummary.errors}`);
     return manifest;
   }
 };
@@ -14178,8 +14290,8 @@ var ImageTransform = class {
    */
   static async toWebp(inputPath, options) {
     const dir = path6.dirname(inputPath);
-    const basename = path6.basename(inputPath, path6.extname(inputPath));
-    const outputPath = path6.join(dir, `${basename}.webp`);
+    const basename2 = path6.basename(inputPath, path6.extname(inputPath));
+    const outputPath = path6.join(dir, `${basename2}.webp`);
     let pipeline = sharp(inputPath);
     const hasSize = options.width !== void 0 && options.height !== void 0;
     if (hasSize) {
@@ -14287,7 +14399,7 @@ var DocumentStream = class {
 };
 
 // src/sdk/models/document-tap/document-tap.ts
-import fs7 from "fs-extra";
+import fs8 from "fs-extra";
 import path7 from "path";
 var BYTES_PER_KB = 1024;
 function formatFileSize(bytes) {
@@ -14297,13 +14409,13 @@ function formatFileSize(bytes) {
 }
 async function getFileSizeLabel(filePath) {
   try {
-    const stat = await fs7.stat(filePath);
+    const stat = await fs8.stat(filePath);
     return formatFileSize(stat.size);
   } catch {
     return "unknown size";
   }
 }
-var logPrefix7 = "[DocumentTap]";
+var logPrefix8 = "[DocumentTap]";
 var DocumentTap = class {
   config;
   outputDir;
@@ -14322,32 +14434,32 @@ var DocumentTap = class {
   }
   async sync() {
     if (!this.target) {
-      throw new Error(`${logPrefix7} No target set. Assign a WhalyDocumentTarget before calling sync().`);
+      throw new Error(`${logPrefix8} No target set. Assign a WhalyDocumentTarget before calling sync().`);
     }
     await this.init();
     if (!this.stream) {
-      throw new Error(`${logPrefix7} No stream set. Assign this.stream in init().`);
+      throw new Error(`${logPrefix8} No stream set. Assign this.stream in init().`);
     }
     const dryRun = isDryRun();
     const dryRunLimit = dryRun ? getDryRunLimit() : void 0;
     if (dryRun) {
-      logger.info(`${logPrefix7} [DRY_RUN] mode active \u2014 skipping API calls and uploads`);
-      await fs7.emptyDir(this.outputDir);
+      logger.info(`${logPrefix8} [DRY_RUN] mode active \u2014 skipping API calls and uploads`);
+      await fs8.emptyDir(this.outputDir);
       if (dryRunLimit !== void 0) {
-        logger.info(`${logPrefix7} [DRY_RUN] Limit: ${dryRunLimit} documents per stream`);
+        logger.info(`${logPrefix8} [DRY_RUN] Limit: ${dryRunLimit} documents per stream`);
       }
     } else if (process.env["DRY_RUN_LIMIT"] !== void 0) {
-      logger.warn(`${logPrefix7} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
+      logger.warn(`${logPrefix8} DRY_RUN_LIMIT is set but DRY_RUN is not active \u2014 limit will be ignored`);
     }
     const tmpDir2 = path7.join(this.outputDir, "tmp");
-    await fs7.ensureDir(tmpDir2);
+    await fs8.ensureDir(tmpDir2);
     try {
       const stream = this.stream;
-      logger.info(`${logPrefix7} Processing stream: ${stream.streamId}`);
+      logger.info(`${logPrefix8} Processing stream: ${stream.streamId}`);
       const sourceEntries = await this.collectSourceEntries(stream, dryRunLimit);
       const existingDocs = dryRun ? [] : await this.target.listExistingDocuments();
       const diff = this.computeDiff(stream, sourceEntries, existingDocs);
-      logger.info(`${logPrefix7} Stream ${stream.streamId} diff: create=${diff.toCreate.length} reupload=${diff.toReupload.length} updateMeta=${diff.toUpdateMetadata.length} delete=${diff.toDelete.length} skip=${diff.skipped.length}`);
+      logger.info(`${logPrefix8} Stream ${stream.streamId} diff: create=${diff.toCreate.length} reupload=${diff.toReupload.length} updateMeta=${diff.toUpdateMetadata.length} delete=${diff.toDelete.length} skip=${diff.skipped.length}`);
       const entries = [];
       const createTasks = diff.toCreate.map((entry) => async () => {
         return this.executeCreate(stream, entry, tmpDir2, dryRun);
@@ -14379,12 +14491,12 @@ var DocumentTap = class {
         streams: [streamManifest],
         summary: streamSummary
       };
-      await fs7.ensureDir(this.outputDir);
-      await fs7.writeJson(path7.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
-      logger.info(`${logPrefix7} Sync complete. Created=${streamSummary.created} Updated=${streamSummary.updated} Reuploaded=${streamSummary.reuploaded} Deleted=${streamSummary.deleted} Skipped=${streamSummary.skipped} Errors=${streamSummary.errors}`);
+      await fs8.ensureDir(this.outputDir);
+      await fs8.writeJson(path7.join(this.outputDir, "manifest.json"), manifest, { spaces: 2 });
+      logger.info(`${logPrefix8} Sync complete. Created=${streamSummary.created} Updated=${streamSummary.updated} Reuploaded=${streamSummary.reuploaded} Deleted=${streamSummary.deleted} Skipped=${streamSummary.skipped} Errors=${streamSummary.errors}`);
       return manifest;
     } finally {
-      await fs7.remove(tmpDir2).catch(() => void 0);
+      await fs8.remove(tmpDir2).catch(() => void 0);
     }
   }
   async collectSourceEntries(stream, limit) {
@@ -14443,21 +14555,21 @@ var DocumentTap = class {
         await this.target.createDocument(stream.streamId, entry, downloadPath);
       } else {
         const inspectPath = path7.join(this.outputDir, stream.streamId, `${safeId}.${entry.extension}`);
-        await fs7.ensureDir(path7.dirname(inspectPath));
-        await fs7.copy(downloadPath, inspectPath);
+        await fs8.ensureDir(path7.dirname(inspectPath));
+        await fs8.copy(downloadPath, inspectPath);
       }
       return { externalId: entry.externalId, fileName: entry.fileName, status: "created" };
     } catch (err) {
       if (err instanceof DocumentDownloadSkipError) {
-        logger.warn(`${logPrefix7} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
+        logger.warn(`${logPrefix8} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
         return { externalId: entry.externalId, fileName: entry.fileName, status: "skipped" };
       }
       const message = err instanceof Error ? err.message : String(err);
       const sizeLabel = await getFileSizeLabel(downloadPath);
-      logger.error(`${logPrefix7} Failed to create ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
+      logger.error(`${logPrefix8} Failed to create ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     } finally {
-      await fs7.remove(downloadPath).catch(() => void 0);
+      await fs8.remove(downloadPath).catch(() => void 0);
     }
   }
   async executeReupload(stream, entry, existingDoc, tmpDir2, dryRun) {
@@ -14469,21 +14581,21 @@ var DocumentTap = class {
         await this.target.reuploadDocument(stream.streamId, existingDoc.id, entry, downloadPath);
       } else {
         const inspectPath = path7.join(this.outputDir, stream.streamId, `${safeId}.${entry.extension}`);
-        await fs7.ensureDir(path7.dirname(inspectPath));
-        await fs7.copy(downloadPath, inspectPath);
+        await fs8.ensureDir(path7.dirname(inspectPath));
+        await fs8.copy(downloadPath, inspectPath);
       }
       return { externalId: entry.externalId, fileName: entry.fileName, status: "reuploaded" };
     } catch (err) {
       if (err instanceof DocumentDownloadSkipError) {
-        logger.warn(`${logPrefix7} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
+        logger.warn(`${logPrefix8} Skipped ${entry.externalId} (${entry.fileName}): ${err.message}`);
         return { externalId: entry.externalId, fileName: entry.fileName, status: "skipped" };
       }
       const message = err instanceof Error ? err.message : String(err);
       const sizeLabel = await getFileSizeLabel(downloadPath);
-      logger.error(`${logPrefix7} Failed to reupload ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
+      logger.error(`${logPrefix8} Failed to reupload ${entry.externalId} (${entry.fileName}, ${sizeLabel}): ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     } finally {
-      await fs7.remove(downloadPath).catch(() => void 0);
+      await fs8.remove(downloadPath).catch(() => void 0);
     }
   }
   async executeMetadataUpdate(entry, existingDoc, dryRun) {
@@ -14494,7 +14606,7 @@ var DocumentTap = class {
       return { externalId: entry.externalId, fileName: entry.fileName, status: "updated" };
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      logger.error(`${logPrefix7} Failed to update metadata for ${entry.externalId}: ${message}`);
+      logger.error(`${logPrefix8} Failed to update metadata for ${entry.externalId}: ${message}`);
       return { externalId: entry.externalId, fileName: entry.fileName, status: "error", error: message };
     }
   }
@@ -14506,7 +14618,7 @@ var DocumentTap = class {
       return { externalId: doc.external_id, fileName: doc.file_name, status: "deleted" };
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      logger.error(`${logPrefix7} Failed to delete ${doc.external_id}: ${message}`);
+      logger.error(`${logPrefix8} Failed to delete ${doc.external_id}: ${message}`);
       return { externalId: doc.external_id, fileName: doc.file_name, status: "error", error: message };
     }
   }
@@ -14526,13 +14638,13 @@ var DocumentTap = class {
 };
 
 // src/sdk/models/document-target/whaly-document-target.ts
-import fs9 from "fs-extra";
+import fs10 from "fs-extra";
 
 // src/services/whaly-document.ts
 var import_form_data = __toESM(require_form_data());
 import axios4 from "axios";
 import axiosRetry3 from "axios-retry";
-import fs8 from "fs";
+import fs9 from "fs";
 import { Storage as Storage2 } from "@google-cloud/storage";
 function enrichAxiosError(err) {
   if (!axios4.isAxiosError(err)) return err instanceof Error ? err : new Error(String(err));
@@ -14543,20 +14655,22 @@ function enrichAxiosError(err) {
   const detail = typeof body === "string" ? body : JSON.stringify(body ?? "");
   return new Error(`${method} ${url} failed with status ${status}: ${detail}`, { cause: err });
 }
-var logPrefix8 = "[WhalyDocumentService]";
+var logPrefix9 = "[WhalyDocumentService]";
 var MAX_RETRIES2 = 10;
 var WhalyDocumentService = class {
   axiosClient;
   objectStorageId;
+  documentSourceId;
   gcsBucket;
   constructor(config) {
     const resolvedKey = getServiceAccountKey(config.serviceAccountKey);
     const resolvedEndpoint = getApiEndpoint(config.apiEndpoint);
     this.objectStorageId = config.objectStorageId;
+    this.documentSourceId = config.documentSourceId;
     const gcsBucketName = process.env["WLY_GCS_BUCKET"];
     if (gcsBucketName) {
       this.gcsBucket = new Storage2().bucket(gcsBucketName);
-      logger.info(`${logPrefix8} Using direct GCS upload to bucket: ${gcsBucketName}`);
+      logger.info(`${logPrefix9} Using direct GCS upload to bucket: ${gcsBucketName}`);
     }
     this.axiosClient = axios4.create({
       baseURL: resolvedEndpoint,
@@ -14577,22 +14691,26 @@ var WhalyDocumentService = class {
         if (retryAfter) {
           const seconds = Number(retryAfter);
           if (!isNaN(seconds) && seconds > 0) {
-            logger.info(`${logPrefix8} Rate limited \u2014 waiting ${seconds}s, attempt ${retryCount}/${MAX_RETRIES2}`);
+            logger.info(`${logPrefix9} Rate limited \u2014 waiting ${seconds}s, attempt ${retryCount}/${MAX_RETRIES2}`);
             return seconds * 1e3;
           }
         }
         const delay = axiosRetry3.exponentialDelay(retryCount);
-        logger.info(`${logPrefix8} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES2}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
+        logger.info(`${logPrefix9} Retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES2}) for ${error.config?.method?.toUpperCase()} ${error.config?.url}`);
         return delay;
       }
     });
   }
-  /** Fetch all documents from the API, handling cursor-based pagination. */
+  /**
+   * Fetch all documents belonging to the configured document source,
+   * handling cursor-based pagination. The `document_source_id` server-side
+   * filter scopes the result to this connector's source only.
+   */
   async listAllDocuments() {
     const documents = [];
     let after;
     while (true) {
-      const params = {};
+      const params = { document_source_id: this.documentSourceId };
       if (after) params["after"] = after;
       const response = await this.axiosClient.get(
         "/v1/documents",
@@ -14603,7 +14721,7 @@ var WhalyDocumentService = class {
       if (!nextAfter) break;
       after = nextAfter;
     }
-    logger.info(`${logPrefix8} Fetched ${documents.length} existing documents from Whaly`);
+    logger.info(`${logPrefix9} Fetched ${documents.length} existing documents from Whaly`);
     return documents;
   }
   /** Upload a file to object storage. Returns storage path and size. */
@@ -14624,11 +14742,11 @@ var WhalyDocumentService = class {
       } catch (err) {
         if (attempt === maxAttempts) throw err;
         const delay = Math.min(1e3 * 2 ** (attempt - 1), 3e4);
-        logger.info(`${logPrefix8} GCS upload retry ${attempt}/${MAX_RETRIES2} in ${delay}ms for ${destinationPath}`);
-        await new Promise((resolve) => setTimeout(resolve, delay));
+        logger.info(`${logPrefix9} GCS upload retry ${attempt}/${MAX_RETRIES2} in ${delay}ms for ${destinationPath}`);
+        await new Promise((resolve2) => setTimeout(resolve2, delay));
       }
     }
-    const stat = await fs8.promises.stat(localFilePath);
+    const stat = await fs9.promises.stat(localFilePath);
     return {
       storage: this.objectStorageId,
       filePath: destinationPath,
@@ -14637,7 +14755,7 @@ var WhalyDocumentService = class {
   }
   async uploadFileViaAPI(destinationPath, localFilePath, fileName) {
     const form = new import_form_data.default();
-    form.append("file", fs8.createReadStream(localFilePath), fileName);
+    form.append("file", fs9.createReadStream(localFilePath), fileName);
     try {
       const response = await this.axiosClient.post(
         `/v1/object-storages/${this.objectStorageId}/upload`,
@@ -14652,19 +14770,26 @@ var WhalyDocumentService = class {
       throw enrichAxiosError(err);
     }
   }
-  /** Create a new document record. */
+  /** Create a new document record, attached to the configured document source. */
   async createDocument(payload) {
     try {
-      const response = await this.axiosClient.post("/v1/documents", payload);
+      const response = await this.axiosClient.post("/v1/documents", {
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
     }
   }
-  /** Update an existing document record. */
+  /** Update an existing document record, keeping it in the configured document source. */
   async updateDocument(id, payload) {
     try {
-      const response = await this.axiosClient.put(`/v1/documents/${id}`, { id, ...payload });
+      const response = await this.axiosClient.put(`/v1/documents/${id}`, {
+        id,
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
@@ -14681,7 +14806,7 @@ var WhalyDocumentService = class {
 };
 
 // src/sdk/models/document-target/whaly-document-target.ts
-var logPrefix9 = "[WhalyDocumentTarget]";
+var logPrefix10 = "[WhalyDocumentTarget]";
 var WhalyDocumentTarget = class {
   config;
   service;
@@ -14700,9 +14825,9 @@ var WhalyDocumentTarget = class {
   async createDocument(streamId, entry, localFilePath) {
     const destinationPath = `${streamId}/${entry.externalId}.${entry.extension}`;
     const fileName = `${entry.externalId}.${entry.extension}`;
-    logger.info(`${logPrefix9} Uploading file for new document: ${entry.externalId}`);
+    logger.info(`${logPrefix10} Uploading file for new document: ${entry.externalId}`);
     const uploadResult = await this.service.uploadFile(destinationPath, localFilePath, fileName);
-    const stat = await fs9.stat(localFilePath);
+    const stat = await fs10.stat(localFilePath);
     await this.service.createDocument({
       file_name: entry.fileName,
       external_id: entry.externalId,
@@ -14717,7 +14842,7 @@ var WhalyDocumentTarget = class {
       storage: uploadResult.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Created document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Created document: ${entry.externalId} (${entry.fileName})`);
   }
   /**
    * Re-upload the file and update the document record.
@@ -14725,9 +14850,9 @@ var WhalyDocumentTarget = class {
   async reuploadDocument(streamId, docId, entry, localFilePath) {
     const destinationPath = `${streamId}/${entry.externalId}.${entry.extension}`;
     const fileName = `${entry.externalId}.${entry.extension}`;
-    logger.info(`${logPrefix9} Re-uploading file for document: ${entry.externalId}`);
+    logger.info(`${logPrefix10} Re-uploading file for document: ${entry.externalId}`);
     const uploadResult = await this.service.uploadFile(destinationPath, localFilePath, fileName);
-    const stat = await fs9.stat(localFilePath);
+    const stat = await fs10.stat(localFilePath);
     await this.service.updateDocument(docId, {
       external_id: entry.externalId,
       file_name: entry.fileName,
@@ -14742,7 +14867,7 @@ var WhalyDocumentTarget = class {
       storage: uploadResult.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Re-uploaded document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Re-uploaded document: ${entry.externalId} (${entry.fileName})`);
   }
   /** Update only the metadata fields (no file re-upload). */
   async updateDocumentMetadata(docId, entry, existingDoc) {
@@ -14760,12 +14885,12 @@ var WhalyDocumentTarget = class {
       storage: existingDoc.storage,
       metadata: entry.metadata ?? {}
     });
-    logger.info(`${logPrefix9} Updated metadata for document: ${entry.externalId} (${entry.fileName})`);
+    logger.info(`${logPrefix10} Updated metadata for document: ${entry.externalId} (${entry.fileName})`);
   }
   /** Delete a document record. */
   async deleteDocument(docId, externalId) {
     await this.service.deleteDocument(docId);
-    logger.info(`${logPrefix9} Deleted document: ${externalId} (id=${docId})`);
+    logger.info(`${logPrefix10} Deleted document: ${externalId} (id=${docId})`);
   }
 };
 
@@ -15104,7 +15229,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
         gracefulExit(1);
       } else {
         logger.error(`StreamId: ${this.streamId} - Error when checking if dataset already exists. Retrying... ${err.message} - ${err.code}`);
-        await new Promise((resolve, reject) => setTimeout(resolve, 1e3 * retryCount));
+        await new Promise((resolve2, reject) => setTimeout(resolve2, 1e3 * retryCount));
         await this.createDatabaseAndSchemaIfNotExists(retryCount + 1);
       }
     }
@@ -15331,8 +15456,8 @@ var BigQueryTarget = class extends ITarget {
 };
 
 // src/targets/cdn/main.ts
-import fs10 from "fs-extra";
-var logPrefix10 = "[CdnAssetTarget]";
+import fs11 from "fs-extra";
+var logPrefix11 = "[CdnAssetTarget]";
 var CdnAssetTarget = class extends AssetTarget {
   cdnService;
   constructor(config) {
@@ -15342,31 +15467,31 @@ var CdnAssetTarget = class extends AssetTarget {
   async shouldSync(entry) {
     const metadata = await this.cdnService.getFileMetadata(getCdnId(this.config.cdnId), entry.destinationPath);
     if (!metadata.exists) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} not in CDN \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} not in CDN \u2192 will sync`);
       return true;
     }
     if (entry.lastModified === void 0) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} source has no lastModified \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} source has no lastModified \u2192 will sync`);
       return true;
     }
     if (metadata.lastModified === null) {
-      logger.debug(`${logPrefix10} ${entry.destinationPath} CDN has no lastModified \u2192 will sync`);
+      logger.debug(`${logPrefix11} ${entry.destinationPath} CDN has no lastModified \u2192 will sync`);
       return true;
     }
     const sourceIsNewer = entry.lastModified > metadata.lastModified;
     if (!sourceIsNewer) {
-      logger.debug(`${logPrefix10} Skipping ${entry.destinationPath} (CDN is up-to-date)`);
+      logger.debug(`${logPrefix11} Skipping ${entry.destinationPath} (CDN is up-to-date)`);
     }
     return sourceIsNewer;
   }
   async uploadAsset(asset) {
-    const fileBuffer = await fs10.readFile(asset.uploadPath);
+    const fileBuffer = await fs11.readFile(asset.uploadPath);
     await this.cdnService.uploadFile(
       getCdnId(this.config.cdnId),
       asset.entry.destinationPath,
       fileBuffer
     );
-    logger.info(`${logPrefix10} Uploaded ${asset.entry.destinationPath} (${fileBuffer.length} bytes)`);
+    logger.info(`${logPrefix11} Uploaded ${asset.entry.destinationPath} (${fileBuffer.length} bytes)`);
   }
 };
 
@@ -15435,6 +15560,7 @@ export {
   GCSStateProvider,
   ITarget,
   ImageTransform,
+  LocalStorageService,
   MIME_TYPES,
   MissingFieldInSchemaError,
   MissingSchemaError,