@whaly/connector-sdk 0.3.11 → 0.3.13

package/dist/index.mjs

@@ -11195,6 +11195,20 @@ var Tap = class {
 
 // src/sdk/models/tap/stream.ts
 import { format as format3 } from "util";
+var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
+var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
+function getProgressLogPercentStep() {
+  const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
+  return Math.min(n, 100);
+}
+function getProgressLogFallbackInterval() {
+  const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
+  return n;
+}
 var Stream = class {
   // Runtime values, can't be overriden
   config;
@@ -11457,6 +11471,9 @@ var Stream = class {
   // Private sync methods:
   async _syncRecords(parent) {
     let rowsSent = 0;
+    let lastLoggedPercent = 0;
+    const progressPercentStep = getProgressLogPercentStep();
+    const progressFallbackInterval = getProgressLogFallbackInterval();
     const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
     let recordSyncedMetrics = [];
     if (this.rowsSyncedMetricsConf.isEnabled() === true) {
@@ -11503,9 +11520,14 @@ var Stream = class {
         logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
         break;
       }
-      if (rowsSent % 1e3 === 0) {
-        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
-        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      if (this.totalRows) {
+        const percent = Math.floor(rowsSent / this.totalRows * 100);
+        if (percent >= lastLoggedPercent + progressPercentStep) {
+          lastLoggedPercent = percent - percent % progressPercentStep;
+          logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
+        }
+      } else if (rowsSent % progressFallbackInterval === 0) {
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
       }
     }
     const stateServiceInst = StateService.getInstance();
@@ -14660,11 +14682,13 @@ var MAX_RETRIES2 = 10;
 var WhalyDocumentService = class {
   axiosClient;
   objectStorageId;
+  documentSourceId;
   gcsBucket;
   constructor(config) {
     const resolvedKey = getServiceAccountKey(config.serviceAccountKey);
     const resolvedEndpoint = getApiEndpoint(config.apiEndpoint);
     this.objectStorageId = config.objectStorageId;
+    this.documentSourceId = config.documentSourceId;
     const gcsBucketName = process.env["WLY_GCS_BUCKET"];
     if (gcsBucketName) {
       this.gcsBucket = new Storage2().bucket(gcsBucketName);
@@ -14699,12 +14723,16 @@ var WhalyDocumentService = class {
       }
     });
   }
-  /** Fetch all documents from the API, handling cursor-based pagination. */
+  /**
+   * Fetch all documents belonging to the configured document source,
+   * handling cursor-based pagination. The `document_source_id` server-side
+   * filter scopes the result to this connector's source only.
+   */
   async listAllDocuments() {
     const documents = [];
     let after;
     while (true) {
-      const params = {};
+      const params = { document_source_id: this.documentSourceId };
       if (after) params["after"] = after;
       const response = await this.axiosClient.get(
         "/v1/documents",
@@ -14764,19 +14792,26 @@ var WhalyDocumentService = class {
       throw enrichAxiosError(err);
     }
   }
-  /** Create a new document record. */
+  /** Create a new document record, attached to the configured document source. */
   async createDocument(payload) {
     try {
-      const response = await this.axiosClient.post("/v1/documents", payload);
+      const response = await this.axiosClient.post("/v1/documents", {
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
     }
   }
-  /** Update an existing document record. */
+  /** Update an existing document record, keeping it in the configured document source. */
   async updateDocument(id, payload) {
     try {
-      const response = await this.axiosClient.put(`/v1/documents/${id}`, { id, ...payload });
+      const response = await this.axiosClient.put(`/v1/documents/${id}`, {
+        id,
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);