@whaly/connector-sdk 0.3.12 → 0.3.13

package/dist/index.js

@@ -11317,6 +11317,20 @@ var Tap = class {
 
 // src/sdk/models/tap/stream.ts
 var import_util2 = require("util");
+var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
+var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
+function getProgressLogPercentStep() {
+  const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
+  return Math.min(n, 100);
+}
+function getProgressLogFallbackInterval() {
+  const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
+  return n;
+}
 var Stream = class {
   // Runtime values, can't be overriden
   config;
@@ -11579,6 +11593,9 @@ var Stream = class {
   // Private sync methods:
   async _syncRecords(parent) {
     let rowsSent = 0;
+    let lastLoggedPercent = 0;
+    const progressPercentStep = getProgressLogPercentStep();
+    const progressFallbackInterval = getProgressLogFallbackInterval();
     const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
     let recordSyncedMetrics = [];
     if (this.rowsSyncedMetricsConf.isEnabled() === true) {
@@ -11625,9 +11642,14 @@ var Stream = class {
         logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
         break;
       }
-      if (rowsSent % 1e3 === 0) {
-        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
-        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      if (this.totalRows) {
+        const percent = Math.floor(rowsSent / this.totalRows * 100);
+        if (percent >= lastLoggedPercent + progressPercentStep) {
+          lastLoggedPercent = percent - percent % progressPercentStep;
+          logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
+        }
+      } else if (rowsSent % progressFallbackInterval === 0) {
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
       }
     }
     const stateServiceInst = StateService.getInstance();