@owox/connectors 0.12.0-next-20251113083123 → 0.12.0-next-20251113104154

package/dist/index.cjs

@@ -3779,21 +3779,9 @@ const XAds = (function() {
         }
       }));
       this.fieldsSchema = XAdsFieldsSchema;
-      this._tweetsCache = /* @__PURE__ */ new Map();
       this._promotedTweetsCache = /* @__PURE__ */ new Map();
       this.BASE_URL = "https://ads-api.x.com/";
     }
-    /**
-     * Returns credential fields for this source
-     */
-    getCredentialFields() {
-      return {
-        ConsumerKey: this.config.ConsumerKey,
-        ConsumerSecret: this.config.ConsumerSecret,
-        AccessToken: this.config.AccessToken,
-        AccessTokenSecret: this.config.AccessTokenSecret
-      };
-    }
     /**
      * Single entry point for *all* fetches.
      * @param {Object} opts
@@ -3836,34 +3824,6 @@ const XAds = (function() {
           throw new ConfigurationError(`Unknown node: ${nodeName}`);
       }
     }
-    /**
-     * Get cached data if all requested fields are present
-     * @param {Map} cache - Cache map to check
-     * @param {string} accountId - Account ID to look up
-     * @param {Array<string>} fields - Required fields
-     * @returns {Array|null} - Cached data or null if not found/invalid
-     * @private
-     */
-    _getCachedData(cache, accountId, fields) {
-      if (!cache.has(accountId)) return null;
-      const cached = cache.get(accountId);
-      const hasAllFields = fields.every((field) => cached.fields.has(field));
-      return hasAllFields ? cached.data : null;
-    }
-    /**
-     * Store data in cache with its fields
-     * @param {Map} cache - Cache map to store in
-     * @param {string} accountId - Account ID as key
-     * @param {Array} data - Data to cache
-     * @param {Array<string>} fields - Fields present in the data
-     * @private
-     */
-    _setCacheData(cache, accountId, data, fields) {
-      cache.set(accountId, {
-        data,
-        fields: new Set(fields)
-      });
-    }
     /**
      * Shared logic for non-time-series endpoints
      */
@@ -3873,23 +3833,9 @@ const XAds = (function() {
       if (missingKeys.length > 0) {
         throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
       }
-      if (nodeName === "promoted_tweets") {
-        const cached = this._getCachedData(this._promotedTweetsCache, accountId, fields);
-        if (cached) {
-          console.log("returning cached promoted_tweets");
-          return cached;
-        }
-        console.log("deleting cached promoted_tweets");
-        this._promotedTweetsCache.delete(accountId);
-      }
-      if (nodeName === "tweets") {
-        const cached = this._getCachedData(this._tweetsCache, accountId, fields);
-        if (cached) {
-          console.log("returning cached tweets");
-          return cached;
-        }
-        console.log("deleting cached tweets");
-        this._tweetsCache.delete(accountId);
+      if (nodeName === "promoted_tweets" && this._promotedTweetsCache.has(accountId)) {
+        console.log(`[XAdsSource] Using cached promoted_tweets for account ${accountId}`);
+        return this._promotedTweetsCache.get(accountId);
       }
       let all = await this._fetchPages({
         accountId,
@@ -3905,10 +3851,8 @@ const XAds = (function() {
         }));
       }
       if (nodeName === "promoted_tweets") {
-        this._setCacheData(this._promotedTweetsCache, accountId, all, fields);
-      }
-      if (nodeName === "tweets") {
-        this._setCacheData(this._tweetsCache, accountId, all, fields);
+        console.log(`[XAdsSource] Fetched promoted_tweets from API for account ${accountId}`);
+        this._promotedTweetsCache.set(accountId, all);
       }
       return all;
     }
@@ -4132,10 +4076,10 @@ const XAds = (function() {
       return "OAuth " + Object.keys(oauth).map((k) => `${encodeURIComponent(k)}="${encodeURIComponent(oauth[k])}"`).join(", ");
     }
     /**
-     * Clear tweet/promoted-tweet caches.
-     * */
-    clearTweetsCache(accountId) {
-      this._tweetsCache.delete(accountId);
+     * Clear cache for a specific account
+     * Called after processing all nodes for an account to free up memory
+     */
+    clearCache(accountId) {
       this._promotedTweetsCache.delete(accountId);
     }
   };
@@ -4159,7 +4103,7 @@ const XAds = (function() {
             fields: fields[nodeName] || []
           });
         }
-        this.source.clearTweetsCache(accountId);
+        this.source.clearCache(accountId);
       }
     }
     /**
@@ -5668,16 +5612,6 @@ const TikTokAds = (function() {
       }
       return processedRecord;
     }
-    /**
-     * Returns credential fields for this source
-     */
-    getCredentialFields() {
-      return {
-        AccessToken: this.config.AccessToken,
-        AppId: this.config.AppId,
-        AppSecret: this.config.AppSecret
-      };
-    }
   };
   var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector3 {
     constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
@@ -9369,18 +9303,6 @@ const RedditAds = (function() {
       }
       return false;
     }
-    /**
-     * Returns credential fields for this source
-     */
-    getCredentialFields() {
-      return {
-        ClientId: this.config.ClientId,
-        ClientSecret: this.config.ClientSecret,
-        RedirectUri: this.config.RedirectUri,
-        RefreshToken: this.config.RefreshToken,
-        UserAgent: this.config.UserAgent
-      };
-    }
     /**
      * Keep only requestedFields plus any schema-required keys.
      * @param {Array<Object>|Object} items - Array of objects or single object
@@ -11271,18 +11193,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
       }));
       this.fieldsSchema = MicrosoftAdsFieldsSchema;
     }
-    /**
-     * Returns credential fields for this source
-     * @returns {Object}
-     */
-    getCredentialFields() {
-      return {
-        DeveloperToken: this.config.DeveloperToken,
-        ClientID: this.config.ClientID,
-        ClientSecret: this.config.ClientSecret,
-        RefreshToken: this.config.RefreshToken
-      };
-    }
     /**
      * Retrieve and store an OAuth access token using the refresh token
      */
@@ -11993,17 +11903,6 @@ const LinkedInPages = (function() {
       this.fieldsSchema = LinkedInPagesFieldsSchema;
       this.BASE_URL = "https://api.linkedin.com/rest/";
     }
-    /**
-     * Returns credential fields for this source
-     * @returns {Object}
-     */
-    getCredentialFields() {
-      return {
-        ClientID: this.config.ClientID,
-        ClientSecret: this.config.ClientSecret,
-        RefreshToken: this.config.RefreshToken
-      };
-    }
     /**
      * Main entry point for fetching data from LinkedIn Pages API
      * @param {string} nodeName - Type of resource to fetch
@@ -13269,17 +13168,6 @@ const LinkedInAds = (function() {
       this.MAX_FIELDS_PER_REQUEST = 20;
       this.BASE_URL = "https://api.linkedin.com/rest/";
     }
-    /**
-     * Returns credential fields for this source
-     * @returns {Object}
-     */
-    getCredentialFields() {
-      return {
-        ClientID: this.config.ClientID,
-        ClientSecret: this.config.ClientSecret,
-        RefreshToken: this.config.RefreshToken
-      };
-    }
     /**
      * Main entry point for fetching data from LinkedIn Ads API
      * @param {string} nodeName - Type of resource to fetch (e.g., adAccounts, adCampaigns)
@@ -21155,16 +21043,6 @@ const CriteoAds = (function() {
       }));
       this.fieldsSchema = CriteoAdsFieldsSchema;
     }
-    /**
-     * Return the credential fields needed for this connector
-     * @returns {Object} Object with credential field names and their config objects
-     */
-    getCredentialFields() {
-      return {
-        ClientId: this.config.ClientId,
-        ClientSecret: this.config.ClientSecret
-      };
-    }
     /**
      * Single entry point for *all* fetches.
      * @param {Object} opts

package/dist/index.js

@@ -3784,21 +3784,9 @@ OPTIONS(description="${this.description}")`;
             }
           }));
           this.fieldsSchema = XAdsFieldsSchema;
-          this._tweetsCache = /* @__PURE__ */ new Map();
           this._promotedTweetsCache = /* @__PURE__ */ new Map();
           this.BASE_URL = "https://ads-api.x.com/";
         }
-        /**
-         * Returns credential fields for this source
-         */
-        getCredentialFields() {
-          return {
-            ConsumerKey: this.config.ConsumerKey,
-            ConsumerSecret: this.config.ConsumerSecret,
-            AccessToken: this.config.AccessToken,
-            AccessTokenSecret: this.config.AccessTokenSecret
-          };
-        }
         /**
          * Single entry point for *all* fetches.
          * @param {Object} opts
@@ -3841,34 +3829,6 @@ OPTIONS(description="${this.description}")`;
               throw new ConfigurationError(`Unknown node: ${nodeName}`);
           }
         }
-        /**
-         * Get cached data if all requested fields are present
-         * @param {Map} cache - Cache map to check
-         * @param {string} accountId - Account ID to look up
-         * @param {Array<string>} fields - Required fields
-         * @returns {Array|null} - Cached data or null if not found/invalid
-         * @private
-         */
-        _getCachedData(cache, accountId, fields) {
-          if (!cache.has(accountId)) return null;
-          const cached = cache.get(accountId);
-          const hasAllFields = fields.every((field) => cached.fields.has(field));
-          return hasAllFields ? cached.data : null;
-        }
-        /**
-         * Store data in cache with its fields
-         * @param {Map} cache - Cache map to store in
-         * @param {string} accountId - Account ID as key
-         * @param {Array} data - Data to cache
-         * @param {Array<string>} fields - Fields present in the data
-         * @private
-         */
-        _setCacheData(cache, accountId, data, fields) {
-          cache.set(accountId, {
-            data,
-            fields: new Set(fields)
-          });
-        }
         /**
          * Shared logic for non-time-series endpoints
          */
@@ -3878,23 +3838,9 @@ OPTIONS(description="${this.description}")`;
           if (missingKeys.length > 0) {
             throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
           }
-          if (nodeName === "promoted_tweets") {
-            const cached = this._getCachedData(this._promotedTweetsCache, accountId, fields);
-            if (cached) {
-              console.log("returning cached promoted_tweets");
-              return cached;
-            }
-            console.log("deleting cached promoted_tweets");
-            this._promotedTweetsCache.delete(accountId);
-          }
-          if (nodeName === "tweets") {
-            const cached = this._getCachedData(this._tweetsCache, accountId, fields);
-            if (cached) {
-              console.log("returning cached tweets");
-              return cached;
-            }
-            console.log("deleting cached tweets");
-            this._tweetsCache.delete(accountId);
+          if (nodeName === "promoted_tweets" && this._promotedTweetsCache.has(accountId)) {
+            console.log(`[XAdsSource] Using cached promoted_tweets for account ${accountId}`);
+            return this._promotedTweetsCache.get(accountId);
           }
           let all = await this._fetchPages({
             accountId,
@@ -3910,10 +3856,8 @@ OPTIONS(description="${this.description}")`;
             }));
           }
           if (nodeName === "promoted_tweets") {
-            this._setCacheData(this._promotedTweetsCache, accountId, all, fields);
-          }
-          if (nodeName === "tweets") {
-            this._setCacheData(this._tweetsCache, accountId, all, fields);
+            console.log(`[XAdsSource] Fetched promoted_tweets from API for account ${accountId}`);
+            this._promotedTweetsCache.set(accountId, all);
           }
           return all;
         }
@@ -4137,10 +4081,10 @@ OPTIONS(description="${this.description}")`;
           return "OAuth " + Object.keys(oauth).map((k) => `${encodeURIComponent(k)}="${encodeURIComponent(oauth[k])}"`).join(", ");
         }
         /**
-         * Clear tweet/promoted-tweet caches.
-         * */
-        clearTweetsCache(accountId) {
-          this._tweetsCache.delete(accountId);
+         * Clear cache for a specific account
+         * Called after processing all nodes for an account to free up memory
+         */
+        clearCache(accountId) {
           this._promotedTweetsCache.delete(accountId);
         }
       };
@@ -4164,7 +4108,7 @@ OPTIONS(description="${this.description}")`;
                 fields: fields[nodeName] || []
               });
             }
-            this.source.clearTweetsCache(accountId);
+            this.source.clearCache(accountId);
           }
         }
         /**
@@ -5673,16 +5617,6 @@ OPTIONS(description="${this.description}")`;
           }
           return processedRecord;
         }
-        /**
-         * Returns credential fields for this source
-         */
-        getCredentialFields() {
-          return {
-            AccessToken: this.config.AccessToken,
-            AppId: this.config.AppId,
-            AppSecret: this.config.AppSecret
-          };
-        }
       };
       var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector2 {
         constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
@@ -9374,18 +9308,6 @@ OPTIONS(description="${this.description}")`;
           }
           return false;
         }
-        /**
-         * Returns credential fields for this source
-         */
-        getCredentialFields() {
-          return {
-            ClientId: this.config.ClientId,
-            ClientSecret: this.config.ClientSecret,
-            RedirectUri: this.config.RedirectUri,
-            RefreshToken: this.config.RefreshToken,
-            UserAgent: this.config.UserAgent
-          };
-        }
         /**
          * Keep only requestedFields plus any schema-required keys.
          * @param {Array<Object>|Object} items - Array of objects or single object
@@ -11276,18 +11198,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
           }));
           this.fieldsSchema = MicrosoftAdsFieldsSchema;
         }
-        /**
-         * Returns credential fields for this source
-         * @returns {Object}
-         */
-        getCredentialFields() {
-          return {
-            DeveloperToken: this.config.DeveloperToken,
-            ClientID: this.config.ClientID,
-            ClientSecret: this.config.ClientSecret,
-            RefreshToken: this.config.RefreshToken
-          };
-        }
         /**
          * Retrieve and store an OAuth access token using the refresh token
          */
@@ -11998,17 +11908,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
           this.fieldsSchema = LinkedInPagesFieldsSchema;
           this.BASE_URL = "https://api.linkedin.com/rest/";
         }
-        /**
-         * Returns credential fields for this source
-         * @returns {Object}
-         */
-        getCredentialFields() {
-          return {
-            ClientID: this.config.ClientID,
-            ClientSecret: this.config.ClientSecret,
-            RefreshToken: this.config.RefreshToken
-          };
-        }
         /**
          * Main entry point for fetching data from LinkedIn Pages API
          * @param {string} nodeName - Type of resource to fetch
@@ -13274,17 +13173,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
           this.MAX_FIELDS_PER_REQUEST = 20;
           this.BASE_URL = "https://api.linkedin.com/rest/";
         }
-        /**
-         * Returns credential fields for this source
-         * @returns {Object}
-         */
-        getCredentialFields() {
-          return {
-            ClientID: this.config.ClientID,
-            ClientSecret: this.config.ClientSecret,
-            RefreshToken: this.config.RefreshToken
-          };
-        }
         /**
          * Main entry point for fetching data from LinkedIn Ads API
          * @param {string} nodeName - Type of resource to fetch (e.g., adAccounts, adCampaigns)
@@ -21160,16 +21048,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
           }));
           this.fieldsSchema = CriteoAdsFieldsSchema;
         }
-        /**
-         * Return the credential fields needed for this connector
-         * @returns {Object} Object with credential field names and their config objects
-         */
-        getCredentialFields() {
-          return {
-            ClientId: this.config.ClientId,
-            ClientSecret: this.config.ClientSecret
-          };
-        }
         /**
          * Single entry point for *all* fetches.
          * @param {Object} opts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@owox/connectors",
-  "version": "0.12.0-next-20251113083123",
+  "version": "0.12.0-next-20251113104154",
   "description": "Connectors and storages for different data sources",
   "license": "MIT",
   "publishConfig": {