@whaly/connector-sdk 0.3.11 → 0.3.13

package/dist/index.d.mts

@@ -1489,6 +1489,12 @@ interface WhalyDocument {
     size_kb: number;
     storage: string;
     metadata: Record<string, string> | null;
+    /**
+     * The document source this document belongs to.
+     * `null` means the org's default source. On create this field is optional —
+     * when omitted the document is attached to the default source.
+     */
+    document_source_id: string | null;
 }
 /** Paginated response from the Whaly Document API. */
 interface WhalyPaginatedResponse<T> {
@@ -1566,6 +1572,16 @@ interface WhalyDocumentServiceConfig {
     serviceAccountKey?: string;
     /** Object storage ID for file uploads. */
     objectStorageId: string;
+    /**
+     * The document source this connector manages. One connector maps to exactly
+     * one source. All operations are scoped to it:
+     *  - `listAllDocuments()` only returns documents belonging to this source,
+     *    so the reconciliation (create / update / **delete**) never touches
+     *    documents from other sources — important because the DocumentTap
+     *    deletes any listed document not present in the source.
+     *  - Created and updated documents are always attached to this source.
+     */
+    documentSourceId: string;
 }
 interface WhalyUploadResult {
     storage: string;
@@ -1575,17 +1591,22 @@ interface WhalyUploadResult {
 declare class WhalyDocumentService {
     private axiosClient;
     readonly objectStorageId: string;
+    readonly documentSourceId: string;
     private gcsBucket;
     constructor(config: WhalyDocumentServiceConfig);
-    /** Fetch all documents from the API, handling cursor-based pagination. */
+    /**
+     * Fetch all documents belonging to the configured document source,
+     * handling cursor-based pagination. The `document_source_id` server-side
+     * filter scopes the result to this connector's source only.
+     */
     listAllDocuments(): Promise<WhalyDocument[]>;
     /** Upload a file to object storage. Returns storage path and size. */
     uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
     private uploadFileViaGCS;
     private uploadFileViaAPI;
-    /** Create a new document record. */
-    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
-    /** Update an existing document record. */
+    /** Create a new document record, attached to the configured document source. */
+    createDocument(payload: Omit<WhalyDocument, "id" | "document_source_id">): Promise<WhalyDocument>;
+    /** Update an existing document record, keeping it in the configured document source. */
     updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
     /** Delete a document record. */
     deleteDocument(id: string): Promise<void>;

package/dist/index.d.ts

@@ -1489,6 +1489,12 @@ interface WhalyDocument {
     size_kb: number;
     storage: string;
     metadata: Record<string, string> | null;
+    /**
+     * The document source this document belongs to.
+     * `null` means the org's default source. On create this field is optional —
+     * when omitted the document is attached to the default source.
+     */
+    document_source_id: string | null;
 }
 /** Paginated response from the Whaly Document API. */
 interface WhalyPaginatedResponse<T> {
@@ -1566,6 +1572,16 @@ interface WhalyDocumentServiceConfig {
     serviceAccountKey?: string;
     /** Object storage ID for file uploads. */
     objectStorageId: string;
+    /**
+     * The document source this connector manages. One connector maps to exactly
+     * one source. All operations are scoped to it:
+     *  - `listAllDocuments()` only returns documents belonging to this source,
+     *    so the reconciliation (create / update / **delete**) never touches
+     *    documents from other sources — important because the DocumentTap
+     *    deletes any listed document not present in the source.
+     *  - Created and updated documents are always attached to this source.
+     */
+    documentSourceId: string;
 }
 interface WhalyUploadResult {
     storage: string;
@@ -1575,17 +1591,22 @@ interface WhalyUploadResult {
 declare class WhalyDocumentService {
     private axiosClient;
     readonly objectStorageId: string;
+    readonly documentSourceId: string;
     private gcsBucket;
     constructor(config: WhalyDocumentServiceConfig);
-    /** Fetch all documents from the API, handling cursor-based pagination. */
+    /**
+     * Fetch all documents belonging to the configured document source,
+     * handling cursor-based pagination. The `document_source_id` server-side
+     * filter scopes the result to this connector's source only.
+     */
     listAllDocuments(): Promise<WhalyDocument[]>;
     /** Upload a file to object storage. Returns storage path and size. */
     uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
     private uploadFileViaGCS;
     private uploadFileViaAPI;
-    /** Create a new document record. */
-    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
-    /** Update an existing document record. */
+    /** Create a new document record, attached to the configured document source. */
+    createDocument(payload: Omit<WhalyDocument, "id" | "document_source_id">): Promise<WhalyDocument>;
+    /** Update an existing document record, keeping it in the configured document source. */
     updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
     /** Delete a document record. */
     deleteDocument(id: string): Promise<void>;

package/dist/index.js

@@ -11317,6 +11317,20 @@ var Tap = class {
 
 // src/sdk/models/tap/stream.ts
 var import_util2 = require("util");
+var DEFAULT_PROGRESS_LOG_PERCENT_STEP = 10;
+var DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL = 1e5;
+function getProgressLogPercentStep() {
+  const raw = optionalEnv("PROGRESS_LOG_PERCENT_STEP", String(DEFAULT_PROGRESS_LOG_PERCENT_STEP));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_PERCENT_STEP;
+  return Math.min(n, 100);
+}
+function getProgressLogFallbackInterval() {
+  const raw = optionalEnv("PROGRESS_LOG_FALLBACK_INTERVAL", String(DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL));
+  const n = parseInt(raw, 10);
+  if (isNaN(n) || n <= 0) return DEFAULT_PROGRESS_LOG_FALLBACK_INTERVAL;
+  return n;
+}
 var Stream = class {
   // Runtime values, can't be overriden
   config;
@@ -11579,6 +11593,9 @@ var Stream = class {
   // Private sync methods:
   async _syncRecords(parent) {
     let rowsSent = 0;
+    let lastLoggedPercent = 0;
+    const progressPercentStep = getProgressLogPercentStep();
+    const progressFallbackInterval = getProgressLogFallbackInterval();
     const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
     let recordSyncedMetrics = [];
     if (this.rowsSyncedMetricsConf.isEnabled() === true) {
@@ -11625,9 +11642,14 @@ var Stream = class {
         logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
         break;
       }
-      if (rowsSent % 1e3 === 0) {
-        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
-        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      if (this.totalRows) {
+        const percent = Math.floor(rowsSent / this.totalRows * 100);
+        if (percent >= lastLoggedPercent + progressPercentStep) {
+          lastLoggedPercent = percent - percent % progressPercentStep;
+          logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far... (${percent}%)`);
+        }
+      } else if (rowsSent % progressFallbackInterval === 0) {
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...`);
       }
     }
     const stateServiceInst = StateService.getInstance();
@@ -14782,11 +14804,13 @@ var MAX_RETRIES2 = 10;
 var WhalyDocumentService = class {
   axiosClient;
   objectStorageId;
+  documentSourceId;
   gcsBucket;
   constructor(config) {
     const resolvedKey = getServiceAccountKey(config.serviceAccountKey);
     const resolvedEndpoint = getApiEndpoint(config.apiEndpoint);
     this.objectStorageId = config.objectStorageId;
+    this.documentSourceId = config.documentSourceId;
     const gcsBucketName = process.env["WLY_GCS_BUCKET"];
     if (gcsBucketName) {
       this.gcsBucket = new import_storage2.Storage().bucket(gcsBucketName);
@@ -14821,12 +14845,16 @@ var WhalyDocumentService = class {
       }
     });
   }
-  /** Fetch all documents from the API, handling cursor-based pagination. */
+  /**
+   * Fetch all documents belonging to the configured document source,
+   * handling cursor-based pagination. The `document_source_id` server-side
+   * filter scopes the result to this connector's source only.
+   */
   async listAllDocuments() {
     const documents = [];
     let after;
     while (true) {
-      const params = {};
+      const params = { document_source_id: this.documentSourceId };
       if (after) params["after"] = after;
       const response = await this.axiosClient.get(
         "/v1/documents",
@@ -14886,19 +14914,26 @@ var WhalyDocumentService = class {
       throw enrichAxiosError(err);
     }
   }
-  /** Create a new document record. */
+  /** Create a new document record, attached to the configured document source. */
   async createDocument(payload) {
     try {
-      const response = await this.axiosClient.post("/v1/documents", payload);
+      const response = await this.axiosClient.post("/v1/documents", {
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);
     }
   }
-  /** Update an existing document record. */
+  /** Update an existing document record, keeping it in the configured document source. */
   async updateDocument(id, payload) {
     try {
-      const response = await this.axiosClient.put(`/v1/documents/${id}`, { id, ...payload });
+      const response = await this.axiosClient.put(`/v1/documents/${id}`, {
+        id,
+        ...payload,
+        document_source_id: this.documentSourceId
+      });
       return response.data.data ?? response.data;
     } catch (err) {
       throw enrichAxiosError(err);