@owox/connectors 0.12.0-next-20251105102829 → 0.12.0-next-20251105123137

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -20,11 +20,6 @@ var HTTP_STATUS = {
20
20
  SERVER_ERROR_MIN: 500,
21
21
  SERVER_ERROR_MAX: 599
22
22
  };
23
- var ENVIRONMENT = {
24
- UNKNOWN: 1,
25
- APPS_SCRIPT: 2,
26
- NODE: 3
27
- };
28
23
  var EXECUTION_STATUS = {
29
24
  IMPORT_IN_PROGRESS: 1,
30
25
  CLEANUP_IN_PROGRESS: 2,
@@ -72,269 +67,11 @@ class HttpRequestException extends AbstractException {
72
67
  }
73
68
  class UnsupportedEnvironmentException extends AbstractException {
74
69
  }
75
- var EnvironmentAdapter = class EnvironmentAdapter2 {
76
- /**
77
- * Mac algorithm constants.
78
- *
79
- * @type {Object}
80
- */
81
- static get MacAlgorithm() {
82
- return {
83
- HMAC_SHA_256: "HMAC_SHA_256",
84
- HMAC_SHA_384: "HMAC_SHA_384",
85
- HMAC_SHA_512: "HMAC_SHA_512",
86
- HMAC_SHA_1: "HMAC_SHA_1",
87
- HMAC_MD5: "HMAC_MD5"
88
- };
89
- }
90
- constructor() {
91
- this.environment = this.getEnvironment();
92
- }
93
- /**
94
- * Get the current environment.
95
- * Detects whether code is running in Google Apps Script or Node.js environment.
96
- *
97
- * @returns {ENVIRONMENT} The detected environment (APPS_SCRIPT, NODE, or UNKNOWN)
98
- * @throws {UnsupportedEnvironmentException} If environment cannot be determined
99
- */
100
- static getEnvironment() {
101
- if (typeof this.environment !== "undefined") {
102
- return this.environment;
103
- }
104
- if (typeof UrlFetchApp !== "undefined") {
105
- this.environment = ENVIRONMENT.APPS_SCRIPT;
106
- } else if (typeof process !== "undefined") {
107
- this.environment = ENVIRONMENT.NODE;
108
- } else {
109
- this.environment = ENVIRONMENT.UNKNOWN;
110
- }
111
- return this.environment;
112
- }
113
- /**
114
- * Fetch data from the given URL.
115
- *
116
- * @param {string} url - The URL to fetch data from.
117
- * @param {Object} options - Options for the fetch request.
118
- * @returns {FetchResponse}
119
- *
120
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
121
- */
122
- static fetch(url, options = {}) {
123
- const env = this.getEnvironment();
124
- if (env === ENVIRONMENT.APPS_SCRIPT) {
125
- const response = UrlFetchApp.fetch(url, options);
126
- return this._wrapAppsScriptResponse(response);
127
- }
128
- if (env === ENVIRONMENT.NODE) {
129
- const method = options.method || "GET";
130
- const response = request(method, url, options);
131
- return this._wrapNodeResponse(response);
132
- }
133
- throw new UnsupportedEnvironmentException("Unsupported environment");
134
- }
135
- /**
136
- * Sleep for the given number of milliseconds.
137
- *
138
- * @param {number} ms - The number of milliseconds to sleep.
139
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
140
- */
141
- static sleep(ms) {
142
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
143
- Utilities.sleep(ms);
144
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
145
- let done = false;
146
- new Promise((resolve) => {
147
- setTimeout(() => {
148
- done = true;
149
- resolve();
150
- }, ms);
151
- });
152
- deasync.loopWhile(() => !done);
153
- } else {
154
- throw new UnsupportedEnvironmentException("Unsupported environment");
155
- }
156
- }
157
- /**
158
- * Format the given date.
159
- *
160
- * @param {Date} date - The date to format.
161
- * @param {string} timezone - The timezone to format the date in.
162
- * @param {string} format - The format to format the date in.
163
- * @returns {string}
164
- *
165
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
166
- */
167
- static formatDate(date, timezone, format) {
168
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
169
- return Utilities.formatDate(date, timezone, format);
170
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
171
- return date.toISOString().split("T")[0];
172
- } else {
173
- throw new UnsupportedEnvironmentException("Unsupported environment");
174
- }
175
- }
176
- /**
177
- * Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
178
- *
179
- * @returns {string} UUID
180
- *
181
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
182
- */
183
- static getUuid() {
184
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
185
- return Utilities.getUuid();
186
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
187
- const crypto = require("node:crypto");
188
- return crypto.randomUUID();
189
- } else {
190
- throw new UnsupportedEnvironmentException("Unsupported environment");
191
- }
192
- }
193
- /**
194
- * Encode the given data to base64.
195
- *
196
- * @param {string} data - The data to encode.
197
- * @returns {string}
198
- *
199
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
200
- */
201
- static base64Encode(data) {
202
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
203
- return Utilities.base64Encode(data);
204
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
205
- return Buffer.from(data).toString("base64");
206
- } else {
207
- throw new UnsupportedEnvironmentException("Unsupported environment");
208
- }
209
- }
210
- /**
211
- * Compute the HMAC signature for the given data.
212
- *
213
- * @param {string} algorithm - The algorithm to use.
214
- * @param {string} data - The data to compute the signature for.
215
- * @param {string} key - The key to use.
216
- * @returns {string}
217
- *
218
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
219
- */
220
- static computeHmacSignature(algorithm, data, key) {
221
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
222
- if (typeof algorithm === "string") {
223
- algorithm = Utilities.MacAlgorithm[algorithm];
224
- }
225
- return Utilities.computeHmacSignature(algorithm, data, key);
226
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
227
- const crypto = require("node:crypto");
228
- const algorithmMap = {
229
- "HMAC_SHA_256": "sha256",
230
- "HMAC_SHA_384": "sha384",
231
- "HMAC_SHA_512": "sha512",
232
- "HMAC_SHA_1": "sha1",
233
- "HMAC_MD5": "md5"
234
- };
235
- const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
236
- const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
237
- return Array.from(buffer);
238
- } else {
239
- throw new UnsupportedEnvironmentException("Unsupported environment");
240
- }
241
- }
242
- /**
243
- * Parse CSV string into array of arrays
244
- *
245
- * @param {string} csvString - The CSV string to parse
246
- * @param {string} [delimiter=','] - The delimiter to use for parsing CSV
247
- * @returns {Array<Array<string>>} Parsed CSV data
248
- * @throws {UnsupportedEnvironmentException} If the environment is not supported
249
- */
250
- static parseCsv(csvString, delimiter = ",") {
251
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
252
- return Utilities.parseCsv(csvString, delimiter);
253
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
254
- return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
255
- const trimmed = cell.trim();
256
- if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
257
- return trimmed.slice(1, -1).replace(/""/g, '"');
258
- }
259
- return trimmed;
260
- }));
261
- } else {
262
- throw new UnsupportedEnvironmentException("Unsupported environment");
263
- }
264
- }
265
- /**
266
- * Unzip a blob/buffer
267
- *
268
- * @param {Blob|Buffer} data - The data to unzip
269
- * @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
270
- * @throws {UnsupportedEnvironmentException} If the environment is not supported
271
- */
272
- static unzip(data) {
273
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
274
- return Utilities.unzip(data);
275
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
276
- const zip = new AdmZip(data);
277
- return zip.getEntries().map((entry) => ({
278
- getDataAsString: () => entry.getData().toString("utf8")
279
- }));
280
- } else {
281
- throw new UnsupportedEnvironmentException("Unsupported environment");
282
- }
283
- }
284
- /**
285
- * Wraps the response from the Apps Script environment.
286
- * Not use directly, only for internal purposes.
287
- *
288
- * @param {Object} response
289
- * @returns {FetchResponse}
290
- */
291
- static _wrapAppsScriptResponse(response) {
292
- return {
293
- getHeaders: () => response.getAllHeaders(),
294
- getAsJson: () => {
295
- try {
296
- return JSON.parse(response.getContentText());
297
- } catch (e) {
298
- throw new Error("Invalid JSON response");
299
- }
300
- },
301
- getContent: () => response.getContent(),
302
- getContentText: () => response.getContentText(),
303
- getBlob: () => response.getBlob(),
304
- getResponseCode: () => response.getResponseCode()
305
- };
306
- }
307
- /**
308
- * Wraps the response from the Node environment.
309
- * Not use directly, only for internal purposes.
310
- *
311
- * @param {Object} response
312
- * @returns {FetchResponse}
313
- */
314
- static _wrapNodeResponse(response) {
315
- const headers = response.headers || {};
316
- const text = response.body ? response.body.toString() : "";
317
- return {
318
- getHeaders: () => headers,
319
- getAsJson: () => {
320
- try {
321
- return JSON.parse(text);
322
- } catch (e) {
323
- throw new Error("Invalid JSON response");
324
- }
325
- },
326
- getContent: () => text,
327
- getContentText: () => text,
328
- getBlob: () => response.body,
329
- getResponseCode: () => response.statusCode
330
- };
331
- }
332
- };
333
70
  class AbstractStorage {
334
71
  //---- constructor -------------------------------------------------
335
72
  /**
336
- * Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
337
- * @param config (object) instance of Sheet
73
+ * Abstract class for storage operations providing common methods for data persistence
74
+ * @param config (object) instance of AbstractConfig
338
75
  * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
339
76
  * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
340
77
  * @param description (string) string with storage description }
@@ -358,6 +95,14 @@ class AbstractStorage {
358
95
  }
359
96
  }
360
97
  //----------------------------------------------------------------
98
+ //---- init --------------------------------------------------------
99
+ /**
100
+ * Initializing storage
101
+ */
102
+ async init() {
103
+ throw new Error("Method init() has to be implemented in a child class of AbstractStorage");
104
+ }
105
+ //----------------------------------------------------------------
361
106
  //---- getUniqueKeyByRecordFields ----------------------------------
362
107
  /**
363
108
  * Calculcating unique key based on this.uniqueKeyColumns
@@ -414,11 +159,12 @@ class AbstractStorage {
414
159
  //----------------------------------------------------------------
415
160
  //---- saveData ----------------------------------------------------
416
161
  /**
417
- * Saving data to a storage. Has to be implemented in
162
+ * Saving data to a storage. Has to be implemented in child class as async method.
418
163
  * @param {data} array of assoc objects with records to save
164
+ * @returns {Promise<void>}
419
165
  */
420
- saveData(data) {
421
- throw new Error("Method saveDate() has to be implemented in a child class of AbstractStorage");
166
+ async saveData(data) {
167
+ throw new Error("Method saveData() has to be implemented in a child class of AbstractStorage");
422
168
  }
423
169
  //----------------------------------------------------------------
424
170
  //---- saveRecordsAddedToBuffer ------------------------------------
@@ -489,16 +235,6 @@ class AbstractStorage {
489
235
  return record;
490
236
  }
491
237
  //----------------------------------------------------------------
492
- //---- areHeadersNeeded --------------------------------------------
493
- /**
494
- * Checks if storage needs headers to be added
495
- * By default returns false, should be overridden in child classes if needed
496
- * @returns {boolean} true if headers need to be added, false otherwise
497
- */
498
- areHeadersNeeded() {
499
- return false;
500
- }
501
- //----------------------------------------------------------------
502
238
  //---- getSelectedFields -------------------------------------------
503
239
  /**
504
240
  * Parse Fields config value and return array of selected field names
@@ -552,7 +288,7 @@ var AbstractSource = class AbstractSource2 {
552
288
  * A Data Source-specific methid is used to fetch new data and return it as an array of objects, where each property of an object corresponds to a column name.
553
289
  * @return data array
554
290
  */
555
- fetchData() {
291
+ async fetchData() {
556
292
  throw new Error("Method fetchData must be implemented in Class inheritor of AbstractSource");
557
293
  }
558
294
  //----------------------------------------------------------------
@@ -577,16 +313,16 @@ var AbstractSource = class AbstractSource2 {
577
313
  * @return {HTTPResponse} The response object from the fetch
578
314
  * @throws {HttpRequestException} After exhausting all retries
579
315
  */
580
- urlFetchWithRetry(url, options) {
316
+ async urlFetchWithRetry(url, options) {
581
317
  for (let attempt = 1; attempt <= this.config.MaxFetchRetries.value; attempt++) {
582
318
  try {
583
- const response = EnvironmentAdapter.fetch(url, { ...options, muteHttpExceptions: true });
584
- return this._validateResponse(response);
319
+ const response = await HttpUtils.fetch(url, { ...options, muteHttpExceptions: true });
320
+ return await this._validateResponse(response);
585
321
  } catch (error) {
586
322
  if (!this._shouldRetry(error, attempt)) {
587
323
  throw error;
588
324
  }
589
- this._waitBeforeRetry(attempt);
325
+ await this._waitBeforeRetry(attempt);
590
326
  }
591
327
  }
592
328
  }
@@ -598,12 +334,12 @@ var AbstractSource = class AbstractSource2 {
598
334
  * @throws {HttpRequestException} If the response indicates an error
599
335
  * @private
600
336
  */
601
- _validateResponse(response) {
337
+ async _validateResponse(response) {
602
338
  const code = response.getResponseCode();
603
339
  if (code >= HTTP_STATUS.SUCCESS_MIN && code <= HTTP_STATUS.SUCCESS_MAX) {
604
340
  return response;
605
341
  }
606
- const errorInfo = this._extractErrorInfo(response);
342
+ const errorInfo = await this._extractErrorInfo(response);
607
343
  throw new HttpRequestException({
608
344
  message: errorInfo.message,
609
345
  statusCode: code,
@@ -617,9 +353,9 @@ var AbstractSource = class AbstractSource2 {
617
353
  * @return {Object} Object containing error message and JSON data if available
618
354
  * @private
619
355
  */
620
- _extractErrorInfo(response) {
356
+ async _extractErrorInfo(response) {
621
357
  var _a, _b;
622
- const text = response.getContentText();
358
+ const text = await response.getContentText();
623
359
  let parsedJson = null;
624
360
  let message = text;
625
361
  try {
@@ -656,10 +392,10 @@ var AbstractSource = class AbstractSource2 {
656
392
  * @param {number} attempt - The current attempt number
657
393
  * @private
658
394
  */
659
- _waitBeforeRetry(attempt) {
395
+ async _waitBeforeRetry(attempt) {
660
396
  const delay = this.calculateBackoff(attempt);
661
397
  console.log(`Retrying after ${Math.round(delay / 1e3)}s...`);
662
- EnvironmentAdapter.sleep(delay);
398
+ await AsyncUtils.delay(delay);
663
399
  }
664
400
  //---- calculateBackoff --------------------------------------------
665
401
  /**
@@ -791,7 +527,7 @@ var AbstractConnector = class AbstractConnector2 {
791
527
  /**
792
528
  * Initiates imports new data from a data source
793
529
  */
794
- run() {
530
+ async run() {
795
531
  try {
796
532
  if (this.config.isInProgress()) {
797
533
  this.config.logMessage("Import is already in progress");
@@ -801,11 +537,7 @@ var AbstractConnector = class AbstractConnector2 {
801
537
  this.config.handleStatusUpdate({ status: EXECUTION_STATUS.IMPORT_IN_PROGRESS });
802
538
  this.config.updateLastImportDate();
803
539
  this.config.logMessage("Start importing new data");
804
- if (this.storage !== null && this.storage.areHeadersNeeded()) {
805
- this.storage.addHeader(this.storage.uniqueKeyColumns);
806
- this.config.logMessage(`Column(s) for unique key was added: ${this.storage.uniqueKeyColumns}`);
807
- }
808
- this.startImportProcess();
540
+ await this.startImportProcess();
809
541
  this.config.logMessage("Import is finished");
810
542
  this.config.handleStatusUpdate({
811
543
  status: EXECUTION_STATUS.IMPORT_DONE
@@ -826,7 +558,7 @@ var AbstractConnector = class AbstractConnector2 {
826
558
  /**
827
559
  * A method for calling from Root script for determining parameters needed to fetch new data.
828
560
  */
829
- startImportProcess() {
561
+ async startImportProcess() {
830
562
  var _a;
831
563
  let startDate = null;
832
564
  let endDate = /* @__PURE__ */ new Date();
@@ -837,10 +569,10 @@ var AbstractConnector = class AbstractConnector2 {
837
569
  return;
838
570
  }
839
571
  endDate.setDate(startDate.getDate() + daysToFetch);
840
- let data = this.source.fetchData(startDate, endDate);
572
+ let data = await this.source.fetchData(startDate, endDate);
841
573
  this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
842
574
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value) === "true") {
843
- this.storage.saveData(data);
575
+ await this.storage.saveData(data);
844
576
  }
845
577
  if (this.runConfig.type === RUN_CONFIG_TYPE.INCREMENTAL) {
846
578
  this.config.updateLastRequstedDate(endDate);
@@ -997,31 +729,12 @@ class AbstractConfig {
997
729
  * @param (object) with config data. Properties are parameters names, values are values
998
730
  */
999
731
  constructor(configData) {
1000
- this.addParameter("Environment", {
1001
- value: AbstractConfig.detectEnvironment(),
1002
- requiredType: "number",
1003
- attributes: [CONFIG_ATTRIBUTES.HIDE_IN_CONFIG_FORM, CONFIG_ATTRIBUTES.ADVANCED]
1004
- });
1005
732
  for (var name in configData) {
1006
733
  this.addParameter(name, configData[name]);
1007
734
  }
1008
735
  return this;
1009
736
  }
1010
737
  //----------------------------------------------------------------
1011
- //---- static helper -------------------------------------------------
1012
- /**
1013
- * Determines the runtime environment
1014
- * @returns {ENVIRONMENT} The detected environment
1015
- */
1016
- static detectEnvironment() {
1017
- if (typeof UrlFetchApp !== "undefined") {
1018
- return ENVIRONMENT.APPS_SCRIPT;
1019
- }
1020
- if (typeof process !== "undefined") {
1021
- return ENVIRONMENT.NODE;
1022
- }
1023
- return ENVIRONMENT.UNKNOWN;
1024
- }
1025
738
  //---- mergeParameters ---------------------------------------------
1026
739
  /**
1027
740
  * Merge configuration to existing config
@@ -1228,11 +941,11 @@ class AbstractConfig {
1228
941
  }
1229
942
  //----------------------------------------------------------------
1230
943
  }
1231
- function processShortLinks(data, { shortLinkField, urlFieldName }) {
944
+ async function processShortLinks(data, { shortLinkField, urlFieldName }) {
1232
945
  if (!Array.isArray(data) || data.length === 0) return data;
1233
946
  const shortLinks = _collectUniqueShortLinks(data, shortLinkField, urlFieldName);
1234
947
  if (shortLinks.length === 0) return data;
1235
- const resolvedShortLinks = _resolveShortLinks(shortLinks);
948
+ const resolvedShortLinks = await _resolveShortLinks(shortLinks);
1236
949
  return _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName);
1237
950
  }
1238
951
  function _collectUniqueShortLinks(data, shortLinkField, urlFieldName) {
@@ -1254,13 +967,11 @@ function _isPotentialShortLink(url) {
1254
967
  if (hasParams) return false;
1255
968
  return /^https:\/\/[^\/]+\/[^\/]+$/.test(url);
1256
969
  }
1257
- function _resolveShortLinks(shortLinks) {
1258
- return shortLinks.map((linkObj) => {
970
+ async function _resolveShortLinks(shortLinks) {
971
+ const promises = shortLinks.map(async (linkObj) => {
1259
972
  try {
1260
- const response = EnvironmentAdapter.fetch(linkObj.originalUrl, {
1261
- method: "GET",
1262
- followRedirects: false,
1263
- muteHttpExceptions: true
973
+ const response = await HttpUtils.fetch(linkObj.originalUrl, {
974
+ method: "GET"
1264
975
  });
1265
976
  const headers = response.getHeaders();
1266
977
  const resolvedUrl = headers.Location || headers.location || linkObj.originalUrl;
@@ -1276,6 +987,7 @@ function _resolveShortLinks(shortLinks) {
1276
987
  };
1277
988
  }
1278
989
  });
990
+ return Promise.all(promises);
1279
991
  }
1280
992
  function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName) {
1281
993
  return data.map((record) => {
@@ -1299,15 +1011,15 @@ function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField,
1299
1011
  var OAuthUtils = {
1300
1012
  /**
1301
1013
  * Universal OAuth access token retrieval method
1302
- *
1014
+ *
1303
1015
  * @param {Object} options - All configuration options
1304
1016
  * @param {Object} options.config - Configuration object containing credentials
1305
1017
  * @param {string} options.tokenUrl - OAuth token endpoint URL
1306
1018
  * @param {Object} options.formData - Form data to send in request body
1307
1019
  * @param {Object} [options.headers] - Request headers
1308
- * @returns {string} - The access token
1020
+ * @returns {Promise<string>} - The access token
1309
1021
  */
1310
- getAccessToken({ config, tokenUrl, formData, headers = {} }) {
1022
+ async getAccessToken({ config, tokenUrl, formData, headers = {} }) {
1311
1023
  const requestHeaders = {
1312
1024
  "Content-Type": "application/x-www-form-urlencoded",
1313
1025
  ...headers
@@ -1320,8 +1032,9 @@ var OAuthUtils = {
1320
1032
  body: Object.entries(formData).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
1321
1033
  };
1322
1034
  try {
1323
- const resp = EnvironmentAdapter.fetch(tokenUrl, options);
1324
- const json = JSON.parse(resp.getContentText());
1035
+ const resp = await HttpUtils.fetch(tokenUrl, options);
1036
+ const text = await resp.getContentText();
1037
+ const json = JSON.parse(text);
1325
1038
  if (json.error) {
1326
1039
  throw new Error(`Token error: ${json.error}`);
1327
1040
  }
@@ -1334,7 +1047,7 @@ var OAuthUtils = {
1334
1047
  },
1335
1048
  /**
1336
1049
  * Get access token using Service Account JWT authentication
1337
- *
1050
+ *
1338
1051
  * @param {Object} options - Configuration options
1339
1052
  * @param {Object} options.config - Configuration object
1340
1053
  * @param {string} options.tokenUrl - Token URL
@@ -1342,7 +1055,7 @@ var OAuthUtils = {
1342
1055
  * @param {string} options.scope - OAuth scope (e.g., "https://www.googleapis.com/auth/adwords")
1343
1056
  * @returns {string} - The access token
1344
1057
  */
1345
- getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
1058
+ async getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
1346
1059
  try {
1347
1060
  const serviceAccountData = JSON.parse(serviceAccountKeyJson);
1348
1061
  const now = Math.floor(Date.now() / 1e3);
@@ -1360,7 +1073,7 @@ var OAuthUtils = {
1360
1073
  grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
1361
1074
  assertion: jwt
1362
1075
  };
1363
- const accessToken = this.getAccessToken({
1076
+ const accessToken = await this.getAccessToken({
1364
1077
  config,
1365
1078
  tokenUrl,
1366
1079
  formData
@@ -1413,6 +1126,68 @@ var OAuthUtils = {
1413
1126
  return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
1414
1127
  }
1415
1128
  };
1129
+ var HttpUtils = class HttpUtils2 {
1130
+ /**
1131
+ * Fetch data from the given URL.
1132
+ *
1133
+ * @param {string} url - The URL to fetch data from.
1134
+ * @param {Object} options - Options for the fetch request.
1135
+ * @returns {Promise<FetchResponse>}
1136
+ */
1137
+ static async fetch(url, options = {}) {
1138
+ const method = options.method || "GET";
1139
+ const fetchOptions = {
1140
+ method: method.toUpperCase(),
1141
+ headers: options.headers || {}
1142
+ };
1143
+ if (options.body) {
1144
+ fetchOptions.body = options.body;
1145
+ }
1146
+ const response = await fetch(url, fetchOptions);
1147
+ return this._wrapNodeResponse(response);
1148
+ }
1149
+ /**
1150
+ * Wraps the response from the Node environment.
1151
+ * Not use directly, only for internal purposes.
1152
+ *
1153
+ * @param {Response} response - Native fetch Response object
1154
+ * @returns {FetchResponse}
1155
+ */
1156
+ static _wrapNodeResponse(response) {
1157
+ let textCache = null;
1158
+ let blobCache = null;
1159
+ const getText = async () => {
1160
+ if (textCache === null) {
1161
+ textCache = await response.text();
1162
+ }
1163
+ return textCache;
1164
+ };
1165
+ const getBlob = async () => {
1166
+ if (blobCache === null) {
1167
+ blobCache = await response.arrayBuffer();
1168
+ }
1169
+ return Buffer.from(blobCache);
1170
+ };
1171
+ const headersObj = {};
1172
+ response.headers.forEach((value, key) => {
1173
+ headersObj[key] = value;
1174
+ });
1175
+ return {
1176
+ getHeaders: () => headersObj,
1177
+ getAsJson: () => getText().then((text) => {
1178
+ try {
1179
+ return JSON.parse(text);
1180
+ } catch (e) {
1181
+ throw new Error("Invalid JSON response");
1182
+ }
1183
+ }),
1184
+ getContent: () => getText(),
1185
+ getContentText: () => getText(),
1186
+ getBlob: () => getBlob(),
1187
+ getResponseCode: () => response.status
1188
+ };
1189
+ }
1190
+ };
1416
1191
  var FormatUtils = {
1417
1192
  /**
1418
1193
  * Universal ID parser: parses comma/semicolon separated string to array of numeric IDs
@@ -1455,6 +1230,102 @@ var FormatUtils = {
1455
1230
  }, {});
1456
1231
  }
1457
1232
  };
1233
+ var FileUtils = class FileUtils2 {
1234
+ /**
1235
+ * Parse CSV string into array of arrays
1236
+ *
1237
+ * @param {string} csvString - The CSV string to parse
1238
+ * @param {string} [delimiter=','] - The delimiter to use for parsing CSV
1239
+ * @returns {Array<Array<string>>} Parsed CSV data
1240
+ */
1241
+ static parseCsv(csvString, delimiter = ",") {
1242
+ return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
1243
+ const trimmed = cell.trim();
1244
+ if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
1245
+ return trimmed.slice(1, -1).replace(/""/g, '"');
1246
+ }
1247
+ return trimmed;
1248
+ }));
1249
+ }
1250
+ /**
1251
+ * Unzip a blob/buffer
1252
+ *
1253
+ * @param {Buffer} data - The data to unzip
1254
+ * @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
1255
+ */
1256
+ static unzip(data) {
1257
+ const zip = new AdmZip(data);
1258
+ return zip.getEntries().map((entry) => ({
1259
+ getDataAsString: () => entry.getData().toString("utf8")
1260
+ }));
1261
+ }
1262
+ };
1263
+ var DateUtils = class DateUtils2 {
1264
+ /**
1265
+ * Format the given date to ISO format (YYYY-MM-DD).
1266
+ *
1267
+ * @param {Date} date - The date to format.
1268
+ * @returns {string} ISO formatted date (YYYY-MM-DD)
1269
+ */
1270
+ static formatDate(date) {
1271
+ return date.toISOString().split("T")[0];
1272
+ }
1273
+ };
1274
+ var CryptoUtils = class CryptoUtils2 {
1275
+ /**
1276
+ * Mac algorithm constants.
1277
+ *
1278
+ * @type {Object}
1279
+ */
1280
+ static get MacAlgorithm() {
1281
+ return {
1282
+ HMAC_SHA_256: "HMAC_SHA_256",
1283
+ HMAC_SHA_384: "HMAC_SHA_384",
1284
+ HMAC_SHA_512: "HMAC_SHA_512",
1285
+ HMAC_SHA_1: "HMAC_SHA_1",
1286
+ HMAC_MD5: "HMAC_MD5"
1287
+ };
1288
+ }
1289
+ /**
1290
+ * Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
1291
+ *
1292
+ * @returns {string} UUID
1293
+ */
1294
+ static getUuid() {
1295
+ const crypto = require("node:crypto");
1296
+ return crypto.randomUUID();
1297
+ }
1298
+ /**
1299
+ * Encode the given data to base64.
1300
+ *
1301
+ * @param {string} data - The data to encode.
1302
+ * @returns {string}
1303
+ */
1304
+ static base64Encode(data) {
1305
+ return Buffer.from(data).toString("base64");
1306
+ }
1307
+ /**
1308
+ * Compute the HMAC signature for the given data.
1309
+ *
1310
+ * @param {string} algorithm - The algorithm to use (e.g., 'HMAC_SHA_256', 'sha256').
1311
+ * @param {string} data - The data to compute the signature for.
1312
+ * @param {string} key - The key to use.
1313
+ * @returns {Array<number>} Byte array of the signature
1314
+ */
1315
+ static computeHmacSignature(algorithm, data, key) {
1316
+ const crypto = require("node:crypto");
1317
+ const algorithmMap = {
1318
+ "HMAC_SHA_256": "sha256",
1319
+ "HMAC_SHA_384": "sha384",
1320
+ "HMAC_SHA_512": "sha512",
1321
+ "HMAC_SHA_1": "sha1",
1322
+ "HMAC_MD5": "md5"
1323
+ };
1324
+ const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
1325
+ const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
1326
+ return Array.from(buffer);
1327
+ }
1328
+ };
1458
1329
  var ConnectorUtils = {
1459
1330
  /**
1460
1331
  * Check if a node is a time series node
@@ -1477,6 +1348,17 @@ var ConnectorUtils = {
1477
1348
  }, {});
1478
1349
  }
1479
1350
  };
1351
+ var AsyncUtils = class AsyncUtils2 {
1352
+ /**
1353
+ * Async delay for the given number of milliseconds.
1354
+ *
1355
+ * @param {number} ms - The number of milliseconds to delay.
1356
+ * @returns {Promise<void>}
1357
+ */
1358
+ static async delay(ms) {
1359
+ return new Promise((resolve) => setTimeout(resolve, ms));
1360
+ }
1361
+ };
1480
1362
  class RunConfigDto {
1481
1363
  constructor(config) {
1482
1364
  this._config = config;
@@ -1888,902 +1770,38 @@ class NodeJsConfig extends AbstractConfig {
1888
1770
  );
1889
1771
  }
1890
1772
  }
1891
- var GoogleSheetsConfig = class GoogleSheetsConfig2 extends AbstractConfig {
1892
- //---- constructor -------------------------------------------------
1893
- constructor(configRange) {
1894
- if (typeof configRange.getA1Notation !== "function") {
1895
- throw new Error(`Unable to create an GoogleSheetsConfig object. The first constructor's parameter must be an SpreadsheetApp.Sheet object`);
1896
- }
1897
- let configObject = {};
1898
- configRange.getValues().forEach((row, index) => {
1899
- var name = row[0].replaceAll(/[^a-zA-Z0-9]/gi, "");
1900
- if (!["", "Parameters", "Status", "Name"].includes(name)) {
1901
- var value = row[1];
1902
- configObject[name] = {};
1903
- if (value || value === 0) {
1904
- configObject[name].value = value;
1905
- }
1906
- if (row[0].slice(-1) == "*") {
1907
- configObject[name].isRequired = true;
1908
- }
1909
- if (["Log", "CurrentStatus", "LastImportDate", "LastRequestedDate", "DestinationSpreadsheet"].includes(name)) {
1910
- configObject[name].cell = configRange.offset(index, 1, 1, 1);
1911
- }
1912
- }
1913
- });
1914
- configObject.configSpreadsheet = configRange.getSheet().getParent();
1915
- configObject.Log.timeZone = configRange.getSheet().getParent().getSpreadsheetTimeZone();
1916
- super(configObject);
1917
- this.mergeParameters({
1918
- MaxRunTimeout: {
1919
- isRequired: true,
1920
- requiredType: "number",
1921
- default: 30
1922
- },
1923
- NotifyByEmail: {
1924
- isRequired: false,
1925
- requiredType: "string",
1926
- default: ""
1927
- },
1928
- NotifyByGoogleChat: {
1929
- isRequired: false,
1930
- requiredType: "string",
1931
- default: ""
1932
- },
1933
- NotifyWhen: {
1934
- isRequired: false,
1935
- requiredType: "string",
1936
- default: "Never"
1937
- }
1938
- });
1939
- }
1940
- //---- handleStatusUpdate -----------------------------------------------
1941
- /**
1942
- * @param {Object} params - Parameters object with status and other properties
1943
- * @param {number} params.status - Status constant
1944
- * @param {string} params.error - Error message for Error status
1945
- */
1946
- handleStatusUpdate({ status, error }) {
1947
- this.manageTimeoutTrigger(status);
1948
- this.updateCurrentStatus(status);
1949
- if (this.shouldSendNotifications(status)) {
1950
- this.sendNotifications({ status, error });
1951
- }
1952
- }
1953
- //----------------------------------------------------------------
1954
- //---- manageTimeoutTrigger ----------------------------------------
1955
- /**
1956
- * Manage timeout trigger based on current status
1957
- * @param {number} status - Status constant
1958
- */
1959
- manageTimeoutTrigger(status) {
1960
- if (status === EXECUTION_STATUS.IMPORT_IN_PROGRESS) {
1961
- this.createTimeoutTrigger();
1962
- } else if (status === EXECUTION_STATUS.IMPORT_DONE || status === EXECUTION_STATUS.ERROR) {
1963
- this.removeTimeoutTrigger();
1964
- }
1965
- }
1966
- //---- getStatusProperties ------------------------------------------
1967
- /**
1968
- * Get all properties for a given status
1969
- * @param {number} status - Status constant
1970
- * @returns {Object} - Object with all status properties
1971
- */
1972
- getStatusProperties(status) {
1973
- switch (status) {
1974
- case EXECUTION_STATUS.IMPORT_IN_PROGRESS:
1975
- return {
1976
- displayText: "Import in progress",
1977
- backgroundColor: "#c9e3f9",
1978
- notificationMessage: "Import is in progress."
1979
- };
1980
- case EXECUTION_STATUS.CLEANUP_IN_PROGRESS:
1981
- return {
1982
- displayText: "CleanUp in progress",
1983
- backgroundColor: "#c9e3f9",
1984
- notificationMessage: "Cleanup is in progress."
1985
- };
1986
- case EXECUTION_STATUS.IMPORT_DONE:
1987
- return {
1988
- displayText: "Done",
1989
- backgroundColor: "#d4efd5",
1990
- notificationMessage: "Import completed successfully."
1991
- };
1992
- case EXECUTION_STATUS.CLEANUP_DONE:
1993
- return {
1994
- displayText: "Done",
1995
- backgroundColor: "#d4efd5",
1996
- notificationMessage: "Cleanup completed successfully."
1997
- };
1998
- case EXECUTION_STATUS.ERROR:
1999
- return {
2000
- displayText: "Error",
2001
- backgroundColor: "#fdd2cf",
2002
- notificationMessage: "Error occurred"
2003
- };
2004
- default:
2005
- throw new Error(`Unknown status constant: ${status}`);
2006
- }
2007
- }
2008
- //----------------------------------------------------------------
2009
- //---- updateCurrentStatus -----------------------------------------
2010
- /**
2011
- * @param {number} status - Status constant
2012
- */
2013
- updateCurrentStatus(status) {
2014
- const statusProps = this.getStatusProperties(status);
2015
- this.CurrentStatus.cell.setValue(statusProps.displayText);
2016
- this.CurrentStatus.cell.setBackground(statusProps.backgroundColor);
2017
- }
2018
- //----------------------------------------------------------------
2019
- //---- updateLastImportDate ----------------------------------------
2020
- /**
2021
- * updating the last import attempt date in a config sheet
2022
- */
2023
- updateLastImportDate() {
2024
- this.LastImportDate.cell.setValue(
2025
- EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss")
2026
- );
2027
- }
2028
- //----------------------------------------------------------------
2029
- //---- updateLastRequstedDate --------------------------------------
2030
- /**
2031
- * Updating the last requested date in a config sheet
2032
- * @param date Date requested date
2033
- */
2034
- updateLastRequstedDate(date) {
2035
- if ("LastRequestedDate" in this && (!this.LastRequestedDate.value || date.getTime() != this.LastRequestedDate.value.getTime())) {
2036
- this.LastRequestedDate.value = new Date(date.getTime());
2037
- this.LastRequestedDate.cell.setValue(EnvironmentAdapter.formatDate(date, this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"));
2038
- }
2039
- }
2040
- //----------------------------------------------------------------
2041
- //---- updateFieldsSheet -------------------------------------------
2042
- /**
2043
- * Updating the content of the fields list to simplify the selection of fields for import
2044
- * @param connector AbstractSource
2045
- * @param sheetName string, Fields by default
2046
- */
2047
- updateFieldsSheet(source, sheetName = "Fields") {
2048
- this.validate();
2049
- var configSheetStorage = new GoogleSheetsStorage(
2050
- this.mergeParameters({
2051
- DestinationSheetName: {
2052
- // @TODO: make sure it would affect destination sheet for import data. CONFIG is an object. Probably we might dublicate it
2053
- value: sheetName
2054
- }
2055
- }),
2056
- ["id"]
2057
- );
2058
- var groups = source.getFieldsSchema();
2059
- var data = [];
2060
- for (var groupName in groups) {
2061
- data.push({
2062
- "id": groupName,
2063
- "✔️": groupName,
2064
- "name": "",
2065
- "description": `=IF( COUNTIFS(A:A, "${groupName} *", B:B, TRUE) > 0, COUNTIFS(A:A, "${groupName} *", B:B, TRUE), "")`
2066
- });
2067
- data.push({ "id": `${groupName} !desc`, "✔️": groups[groupName].description });
2068
- data.push({ "id": `${groupName} !doc`, "✔️": groups[groupName].documentation });
2069
- if (groups[groupName].fields) {
2070
- for (var fieldName in groups[groupName].fields) {
2071
- data.push({
2072
- "id": groupName + " " + fieldName,
2073
- "name": fieldName,
2074
- "description": groups[groupName].fields[fieldName].description
2075
- });
2076
- }
2077
- data.push({ "id": `${groupName} zzz_separator` });
2078
- }
2079
- }
2080
- for (var groupName in groups) {
2081
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2082
- if (groupRow !== null) {
2083
- let depth = configSheetStorage.SHEET.getRowGroupDepth(groupRow.rowIndex + 5);
2084
- if (depth > 0) {
2085
- configSheetStorage.SHEET.getRowGroup(groupRow.rowIndex + 5, depth).remove();
2086
- }
2087
- }
2088
- }
2089
- configSheetStorage.saveData(data);
2090
- configSheetStorage.SHEET.sort(configSheetStorage.getColumnIndexByName("id"));
2091
- configSheetStorage.loadDataFromSheet();
2092
- configSheetStorage.SHEET.hideColumns(configSheetStorage.getColumnIndexByName("id"));
2093
- configSheetStorage.SHEET.setColumnWidth(configSheetStorage.getColumnIndexByName("✔️"), 25);
2094
- configSheetStorage.SHEET.autoResizeColumn(3);
2095
- var checkboxRule = SpreadsheetApp.newDataValidation().requireCheckbox().build();
2096
- for (var groupName in groups) {
2097
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2098
- var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 2}`);
2099
- range.setFontWeight("bold").setBackground("black").setFontColor("white");
2100
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": `${groupName} !desc` });
2101
- var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 3}`);
2102
- range.setBackground("#f3f3f3");
2103
- var checkboxesColumnIndex = configSheetStorage.getColumnIndexByName("✔️");
2104
- var groupNameRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2105
- var groupFieldsCount = this.getFieldsCountByGroupName(configSheetStorage, groupName);
2106
- configSheetStorage.SHEET.getRange(groupNameRow.rowIndex + 5, checkboxesColumnIndex, groupFieldsCount).setDataValidation(checkboxRule);
2107
- configSheetStorage.SHEET.getRange(`${groupNameRow.rowIndex + 5}:${groupNameRow.rowIndex + 4 + groupFieldsCount}`).shiftRowGroupDepth(1);
2108
- }
2109
- configSheetStorage.SHEET.collapseAllRowGroups();
2110
- }
2111
- //----------------------------------------------------------------
2112
- //---- getFieldsCountByGroupName -----------------------------------
2113
- /**
2114
- * @param GoogleSheetsConfig object
2115
- * @param groupName string name of the group
2116
- * @return integer number of fields of requested group
2117
- */
2118
- getFieldsCountByGroupName(configSheetStorage, groupName) {
2119
- return Object.values(configSheetStorage.values).filter((element) => element.name && element.id.startsWith(`${groupName} `)).length;
2120
- }
2121
- //----------------------------------------------------------------
2122
- //---- isInProgress ------------------------------------------------
2123
- /**
2124
- * Checking current status if it is in progress or not
2125
- * @return boolean true is process in progress
2126
- */
2127
- isInProgress() {
2128
- let isInProgress = null;
2129
- if (this.CurrentStatus.cell.getValue().indexOf("progress") !== -1) {
2130
- let diff = (/* @__PURE__ */ new Date() - new Date(EnvironmentAdapter.formatDate(this.LastImportDate.cell.getValue(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"))) / (1e3 * 60);
2131
- isInProgress = diff < this.MaxRunTimeout.value;
2132
- } else {
2133
- isInProgress = false;
2134
- }
2135
- return isInProgress;
2136
- }
2137
- //----------------------------------------------------------------
2138
- //---- addWarningToCurrentStatus -----------------------------------
2139
- addWarningToCurrentStatus() {
2140
- this.CurrentStatus.cell.setBackground("#fff0c4");
2141
- }
2142
- //----------------------------------------------------------------
2143
- //---- validate ----------------------------------------------------
2144
- /**
2145
- * validating if google sheets config is correct
2146
- */
2147
- validate() {
2148
- let scriptTimeZone = Session.getScriptTimeZone();
2149
- if (scriptTimeZone != "Etc/UTC") {
2150
- throw new Error(`The Apps Script time zone must be set to UTC to avoid confusion with dates. Currently, it is set to ${scriptTimeZone} instead. Update the time zone in Project Settings`);
2151
- }
2152
- super.validate();
2153
- }
2154
- //----------------------------------------------------------------
2155
- //---- logMessage --------------------------------------------------
2156
- /**
2157
- * @param string message to Log
2158
- */
2159
- logMessage(message, removeExistingMessage = false) {
2160
- console.log(message);
2161
- let formattedDate = EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss");
2162
- let currentLog = removeExistingMessage ? "" : this.Log.cell.getValue();
2163
- currentLog ? currentLog += "\n" : "";
2164
- let emoji = "☑️ ";
2165
- let match;
2166
- if (match = message.match(new RegExp("^(\\p{Emoji_Presentation}|\\p{Emoji}\\uFE0F|\\p{Extended_Pictographic})\\s+", "u"))) {
2167
- emoji = match[0];
2168
- message = message.slice(2).trim();
2169
- }
2170
- this.Log.cell.setValue(
2171
- `${currentLog}${emoji}${formattedDate}: ${message}`
2172
- );
2173
- this.updateLastImportDate();
2174
- }
2175
- showCredentialsDialog(source) {
2176
- const ui = SpreadsheetApp.getUi();
2177
- const template = HtmlService.createTemplateFromFile("Views/credentials-input-dialog");
2178
- template.source = source;
2179
- const html = template.evaluate().setWidth(400).setHeight(450);
2180
- ui.showModalDialog(html, `${source.constructor.name} Credentials`);
2181
- }
2182
- showManualBackfillDialog(source) {
2183
- const ui = SpreadsheetApp.getUi();
2184
- const template = HtmlService.createTemplateFromFile("Views/manual-backfill-dialog");
2185
- template.source = source;
2186
- const html = template.evaluate().setWidth(600).setHeight(300);
2187
- ui.showModalDialog(html, "Manual Backfill");
2188
- }
2189
- //---- sendNotifications -------------------------------------------
2190
- /**
2191
- * Send notifications based on configuration settings
2192
- * @param {Object} params - Parameters object
2193
- * @param {string} params.status - Current status value
2194
- * @param {string} params.error - Error message for Error status
2195
- */
2196
- sendNotifications({ status, error }) {
2197
- try {
2198
- const { title, messageWithDetails } = this.prepareNotificationContent({ status, error });
2199
- if (this.NotifyByEmail && this.NotifyByEmail.value && this.NotifyByEmail.value.trim()) {
2200
- EmailNotification.send({
2201
- to: this.NotifyByEmail.value,
2202
- subject: title,
2203
- message: messageWithDetails
2204
- });
2205
- }
2206
- if (this.NotifyByGoogleChat && this.NotifyByGoogleChat.value && this.NotifyByGoogleChat.value.trim()) {
2207
- GoogleChatNotification.send({
2208
- webhookUrl: this.NotifyByGoogleChat.value.trim(),
2209
- message: messageWithDetails
2210
- });
2211
- }
2212
- } catch (error2) {
2213
- this.logMessage(`⚠️ Notification error: ${error2.message}`);
2214
- }
2215
- }
2216
- //----------------------------------------------------------------
2217
- //---- prepareNotificationContent ----------------------------------
2218
- /**
2219
- * Prepare notification title and message content
2220
- * @param {Object} params - Parameters object
2221
- * @param {number} params.status - Status constant
2222
- * @param {string} params.error - Error message for Error status
2223
- * @returns {Object} - Object with title and messageWithDetails properties
2224
- */
2225
- prepareNotificationContent({ status, error }) {
2226
- const documentName = this.configSpreadsheet.getName();
2227
- const documentUrl = this.configSpreadsheet.getUrl();
2228
- const { displayText, notificationMessage } = this.getStatusProperties(status);
2229
- const title = `${documentName} - Status: ${displayText}`;
2230
- return {
2231
- title,
2232
- messageWithDetails: `${title}
2233
- ${documentUrl}
2234
-
2235
- ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}` : ""}`
2236
- };
2237
- }
2238
- //----------------------------------------------------------------
2239
- //---- shouldSendNotifications -------------------------------------
2240
- /**
2241
- * Determine if notifications should be sent based on status and filter setting
2242
- * @param {number} status - Status constant
2243
- * @returns {boolean} - True if notifications should be sent
2244
- */
2245
- shouldSendNotifications(status) {
2246
- var _a;
2247
- const notifyWhen = (_a = this.NotifyWhen) == null ? void 0 : _a.value;
2248
- switch (notifyWhen) {
2249
- case "On error":
2250
- return status === EXECUTION_STATUS.ERROR;
2251
- case "On success":
2252
- return status === EXECUTION_STATUS.IMPORT_DONE;
2253
- case "Always":
2254
- return status === EXECUTION_STATUS.ERROR || status === EXECUTION_STATUS.IMPORT_DONE;
2255
- case "Never":
2256
- case "":
2257
- default:
2258
- return false;
2259
- }
2260
- }
2261
- //----------------------------------------------------------------
2262
- //---- createTimeoutTrigger ----------------------------------------
2263
- createTimeoutTrigger() {
2264
- this.removeTimeoutTrigger();
2265
- ScriptApp.newTrigger("checkForTimeout").timeBased().after((this.MaxRunTimeout.value * 2 + 1) * 60 * 1e3).create();
2266
- }
2267
- //----------------------------------------------------------------
2268
- //---- removeTimeoutTrigger ----------------------------------------
2269
- removeTimeoutTrigger() {
2270
- const triggers = ScriptApp.getProjectTriggers();
2271
- let removedCount = 0;
2272
- triggers.forEach((trigger) => {
2273
- if (trigger.getHandlerFunction() === "checkForTimeout") {
2274
- ScriptApp.deleteTrigger(trigger);
2275
- removedCount++;
2276
- }
2277
- });
2278
- console.log(`[TimeoutTrigger] ${removedCount > 0 ? `Removed ${removedCount} timeout trigger(s)` : "No timeout triggers found to remove"}`);
2279
- }
2280
- //----------------------------------------------------------------
2281
- //---- checkForTimeout ---------------------------------------------
2282
- checkForTimeout() {
2283
- if (!this.isInProgress()) {
2284
- console.log("[TimeoutTrigger] Status is NOT in progress, setting to Error and sending notification");
2285
- this.handleStatusUpdate({
2286
- status: EXECUTION_STATUS.ERROR,
2287
- error: "Import was interrupted (likely due to timeout)"
2288
- });
2289
- } else {
2290
- console.log("[TimeoutTrigger] Status is still in progress");
2291
- }
2292
- }
2293
- //----------------------------------------------------------------
2294
- };
2295
- var GoogleChatNotification = class GoogleChatNotification2 {
2296
- /**
2297
- * Send Google Chat notification
2298
- * @param {Object} params - Parameters object
2299
- * @param {string} params.webhookUrl - Google Chat webhook URL
2300
- * @param {string} params.message - Formatted notification message
2301
- */
2302
- static send(params) {
2303
- const { webhookUrl, message } = params;
2304
- if (!webhookUrl || !webhookUrl.trim()) {
2305
- return;
2306
- }
2307
- try {
2308
- const response = EnvironmentAdapter.fetch(webhookUrl.trim(), {
2309
- method: "POST",
2310
- headers: {
2311
- "Content-Type": "application/json; charset=UTF-8"
2312
- },
2313
- payload: JSON.stringify({ text: message })
2314
- });
2315
- if (response.getResponseCode() === 200) {
2316
- console.log("Google Chat notification sent successfully");
2317
- } else {
2318
- console.error(`Google Chat notification failed with status: ${response.getResponseCode()}`);
2319
- }
2320
- } catch (error) {
2321
- console.error("Failed to send Google Chat notification:", error);
2322
- }
2323
- }
2324
- };
2325
- var EmailNotification = class EmailNotification2 {
2326
- /**
2327
- * Send email notification
2328
- * @param {Object} params - Parameters object
2329
- * @param {string} params.to - Email address(es) to send to (can be multiple separated by commas)
2330
- * @param {string} params.subject - Email subject line
2331
- * @param {string} params.message - Formatted notification message
2332
- */
2333
- static send(params) {
2334
- const { to, subject, message } = params;
2335
- if (!to || !to.trim()) {
2336
- return;
2337
- }
2338
- try {
2339
- const emailAddresses = to.split(",").map((email) => email.trim()).filter((email) => email.length > 0).filter((email) => this.isValidEmail(email)).join(",");
2340
- if (!emailAddresses) {
2341
- console.log("Email notification skipped: no valid email addresses found");
2342
- return;
2343
- }
2344
- MailApp.sendEmail(
2345
- emailAddresses,
2346
- subject,
2347
- message,
2348
- { noReply: true }
2349
- );
2350
- console.log(`Email notification sent successfully to: ${emailAddresses}`);
2351
- } catch (error) {
2352
- console.error(`Failed to send email notification: ${error.message}`);
2353
- }
2354
- }
2355
- /**
2356
- * Validate email address format
2357
- * @param {string} email - Email address to validate
2358
- * @returns {boolean} - True if email is valid
2359
- */
2360
- static isValidEmail(email) {
2361
- const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
2362
- const isValid = emailRegex.test(email);
2363
- if (!isValid) {
2364
- console.log(`Invalid email address skipped: ${email}`);
2365
- }
2366
- return isValid;
2367
- }
2368
- };
2369
1773
  const Core = {
2370
1774
  AbstractException,
2371
1775
  HttpRequestException,
2372
1776
  UnsupportedEnvironmentException,
2373
- EnvironmentAdapter,
2374
1777
  AbstractStorage,
2375
1778
  AbstractSource,
2376
1779
  AbstractRunConfig,
2377
1780
  AbstractConnector,
2378
1781
  AbstractConfig,
1782
+ HttpUtils,
1783
+ FileUtils,
1784
+ DateUtils,
1785
+ CryptoUtils,
1786
+ AsyncUtils,
2379
1787
  RunConfigDto,
2380
1788
  SourceConfigDto,
2381
1789
  StorageConfigDto,
2382
1790
  ConfigDto,
2383
1791
  NodeJsConfig,
2384
- GoogleSheetsConfig,
2385
- GoogleChatNotification,
2386
- EmailNotification,
2387
1792
  HTTP_STATUS,
2388
- ENVIRONMENT,
2389
1793
  EXECUTION_STATUS,
2390
1794
  RUN_CONFIG_TYPE,
2391
1795
  CONFIG_ATTRIBUTES
2392
1796
  };
2393
- const GoogleSheets = (function() {
2394
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2395
- var GoogleSheetsStorage2 = class GoogleSheetsStorage extends AbstractStorage2 {
2396
- //---- constructor -------------------------------------------------
2397
- /**
2398
- * Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
2399
- * @param config (object) instance of AbscractConfig
2400
- * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2401
- * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2402
- */
2403
- constructor(config, uniqueKeyColumns, schema = null) {
2404
- super(
2405
- config.mergeParameters({
2406
- CleanUpToKeepWindow: {
2407
- requiredType: "number"
2408
- },
2409
- DestinationSheetName: {
2410
- isRequired: true,
2411
- default: "Data"
2412
- }
2413
- }),
2414
- uniqueKeyColumns,
2415
- schema
2416
- );
2417
- this.SHEET = this.getDestinationSheet(config);
2418
- this.loadDataFromSheet();
2419
- }
2420
- //----------------------------------------------------------------
2421
- //---- loadDataFromSheet -------------------------------------------
2422
- /**
2423
- * reading Data from the source sheet and loading it to this.values
2424
- */
2425
- loadDataFromSheet() {
2426
- const values = this.SHEET.getDataRange().getValues();
2427
- this.columnNames = values.shift();
2428
- if (this.uniqueKeyColumns.some((column) => !this.columnNames.includes(column))) {
2429
- throw new Error(`Sheet '${this.SHEET.getName()}' is missing one the folling columns required for unique key: column '${this.uniqueKeyColumns}'`);
2430
- }
2431
- this.values = values.reduce((acc, row, rowIndex) => {
2432
- const uniqueKey = this.uniqueKeyColumns.reduce((accumulator, columnName) => {
2433
- let index = this.columnNames.indexOf(columnName);
2434
- accumulator += `|${row[index]}`;
2435
- return accumulator;
2436
- }, []);
2437
- acc[uniqueKey] = {
2438
- ...this.columnNames.reduce((obj, col, colIndex) => ({
2439
- ...obj,
2440
- [col]: row[colIndex]
2441
- }), {}),
2442
- rowIndex
2443
- // Add row index for reference
2444
- };
2445
- return acc;
2446
- }, {});
2447
- this.addedRecordsBuffer = [];
2448
- }
2449
- //----------------------------------------------------------------
2450
- //---- getDestinationSheet -----------------------------------------
2451
- /**
2452
- * @param object destination Spreadsheet Config
2453
- * @param object destination Sheet Name Config
2454
- * @return Sheet object for data Destination
2455
- */
2456
- getDestinationSheet(config) {
2457
- if (!this.SHEET) {
2458
- if (!config.DestinationSpreadsheet || !config.DestinationSpreadsheet.value) {
2459
- config.DestinationSpreadsheet = { "spreadsheet": config.configSpreadsheet };
2460
- } else {
2461
- let match = config.DestinationSpreadsheet.value.match(/\/d\/([a-zA-Z0-9-_]+)/);
2462
- if (match && match[1]) {
2463
- config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match[1]);
2464
- } else {
2465
- let match2 = config.DestinationSpreadsheet.cell.getRichTextValue().getLinkUrl().match(/\/d\/([a-zA-Z0-9-_]+)/);
2466
- if (match2 && match2[1]) {
2467
- config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match2[1]);
2468
- } else {
2469
- throw new Error(`Destination Spreadsheet must be specified in config either by Spreadsheet Id or by a link to spreadsheet`);
2470
- }
2471
- }
2472
- }
2473
- if (config.DestinationSpreadsheet.spreadsheet == null) {
2474
- this.config.logMessage(`Cannot load destination sheet document`);
2475
- }
2476
- config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.getSheetByName(
2477
- config.DestinationSheetName.value
2478
- );
2479
- if (!config.DestinationSpreadsheet.sheet) {
2480
- config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.insertSheet(
2481
- config.DestinationSheetName.value,
2482
- config.DestinationSpreadsheet.spreadsheet.getSheets().length
2483
- );
2484
- this.config.logMessage(`Sheet '${config.DestinationSheetName.value}' was created.`);
2485
- }
2486
- this.SHEET = config.DestinationSpreadsheet.sheet;
2487
- if (this.isEmpty()) {
2488
- this.addHeader(this.uniqueKeyColumns);
2489
- this.config.logMessage(`Columns for unique keys were added to '${config.DestinationSheetName.value}' sheet.`);
2490
- }
2491
- }
2492
- return this.SHEET;
2493
- }
2494
- //----------------------------------------------------------------
2495
- //---- addRecord ---------------------------------------------------
2496
- /**
2497
- * Checking if record exists by id
2498
- * @param object {record} object with record data
2499
- * @param Boolean {useBuffer}: Set to `false` if the record must be saved instantly, or `true` to save it later using `this.saveAddedRecordsFromBuffer()`.
2500
- * @return object {record} with added rowIndex property
2501
- */
2502
- addRecord(record, useBuffer = false) {
2503
- record = this.stringifyNeastedFields(record);
2504
- let uniqueKey = this.getUniqueKeyByRecordFields(record);
2505
- if (useBuffer) {
2506
- this.addedRecordsBuffer[uniqueKey] = record;
2507
- } else {
2508
- let data = this.columnNames.map((key) => record[key] || "");
2509
- this.SHEET.appendRow(data);
2510
- record.rowIndex = this.SHEET.getLastRow() - 2;
2511
- this.values[uniqueKey] = record;
2512
- }
2513
- return record;
2514
- }
2515
- //----------------------------------------------------------------
2516
- //---- saveData ----------------------------------------------------
2517
- /**
2518
- * Saving data to a storage
2519
- * @param {data} array of assoc objects with records to save
2520
- */
2521
- saveData(data) {
2522
- var recordsAdded = 0;
2523
- var recordsUpdated = 0;
2524
- data.map((row) => {
2525
- let newFields = Object.keys(row).filter((column2) => !this.columnNames.includes(column2));
2526
- for (var column in newFields) {
2527
- this.addColumn(newFields[column], this.columnNames.length + 1);
2528
- this.config.logMessage(`Column '${newFields[column]}' was added to '${this.SHEET.getName()}' sheet`);
2529
- }
2530
- if (this.isRecordExists(row)) {
2531
- if (this.updateRecord(row)) {
2532
- recordsUpdated++;
2533
- }
2534
- } else {
2535
- this.addRecord(row, true);
2536
- recordsAdded += this.saveRecordsAddedToBuffer(100);
2537
- }
2538
- });
2539
- recordsAdded += this.saveRecordsAddedToBuffer(0);
2540
- if (recordsAdded > 0) {
2541
- this.config.logMessage(`${recordsAdded} records were added`);
2542
- }
2543
- if (recordsUpdated > 0) {
2544
- this.config.logMessage(`${recordsUpdated} records were updated`);
2545
- }
2546
- }
2547
- //----------------------------------------------------------------
2548
- //---- saveRecordsAddedToBuffer ------------------------------------
2549
- /**
2550
- * Add records from buffer to a sheet
2551
- * @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
2552
- */
2553
- saveRecordsAddedToBuffer(maxBufferSize = 0) {
2554
- let recordsAdded = 0;
2555
- let bufferSize = Object.keys(this.addedRecordsBuffer).length;
2556
- if (bufferSize && bufferSize >= maxBufferSize) {
2557
- let startIndex = this.SHEET.getLastRow() - 2;
2558
- let index = 1;
2559
- let data = [];
2560
- for (var uniqueKey in this.addedRecordsBuffer) {
2561
- let record = this.addedRecordsBuffer[uniqueKey];
2562
- record.rowIndex = startIndex + index++;
2563
- this.values[uniqueKey] = record;
2564
- data.push(this.columnNames.map((key) => record[key] || ""));
2565
- }
2566
- this.SHEET.getRange(startIndex + 3, 1, data.length, data[0].length).setValues(data);
2567
- recordsAdded = bufferSize;
2568
- this.addedRecordsBuffer = {};
2569
- }
2570
- return recordsAdded;
2571
- }
2572
- //----------------------------------------------------------------
2573
- //---- updateRecord ------------------------------------------------
2574
- /**
2575
- * Update content of an existing record
2576
- * @param object {record} object with record data
2577
- * @return boolean Returns true if the record was updated; otherwise, returns false
2578
- */
2579
- updateRecord(record) {
2580
- record = this.stringifyNeastedFields(record);
2581
- let uniqueKey = this.getUniqueKeyByRecordFields(record);
2582
- var existingRecord = this.getRecordByUniqueKey(uniqueKey);
2583
- var isRecordUpdated = false;
2584
- this.columnNames.forEach((columnName, columnIndex) => {
2585
- if (columnName in record && !this.areValuesEqual(record[columnName], existingRecord[columnName])) {
2586
- console.log(`${uniqueKey}: ${existingRecord[columnName]} ${typeof existingRecord[columnName]} → ${record[columnName]} ${typeof record[columnName]}`);
2587
- this.SHEET.getRange(existingRecord.rowIndex + 2, columnIndex + 1, 1, 1).setValue(record[columnName]);
2588
- existingRecord[columnName] = record[columnName];
2589
- isRecordUpdated = true;
2590
- }
2591
- });
2592
- return isRecordUpdated;
2593
- }
2594
- //----------------------------------------------------------------
2595
- //---- deleteRecord ------------------------------------------------
2596
- /**
2597
- * Delete record from a sheet
2598
- * @param uniqueKey {string} unique key of the record to delete
2599
- */
2600
- deleteRecord(uniqueKey) {
2601
- if (!(uniqueKey in this.values)) {
2602
- throw new Error(`Unable to delete the record with ID ${uniqueKey} because it was not found`);
2603
- } else if (!("rowIndex" in this.values[uniqueKey])) {
2604
- throw new Error(`Unable to delete the record with ID ${uniqueKey} because it does not have a rowIndex`);
2605
- } else {
2606
- let rowIndex = this.values[uniqueKey].rowIndex;
2607
- this.SHEET.deleteRow(rowIndex + 2);
2608
- for (uniqueKey in this.values) {
2609
- if (this.values[uniqueKey].rowIndex > rowIndex) {
2610
- this.values[uniqueKey].rowIndex--;
2611
- }
2612
- }
2613
- }
2614
- }
2615
- //----------------------------------------------------------------
2616
- //---- addHeader ---------------------------------------------------
2617
- /**
2618
- * Adding header to sheet
2619
- * @param target Sheet
2620
- * @param array column names to be added
2621
- */
2622
- addHeader(columnNames) {
2623
- columnNames.forEach((columnName, index) => {
2624
- this.addColumn(columnName, index + 1);
2625
- });
2626
- this.SHEET.getRange("1:1").setBackground("#f3f3f3").setHorizontalAlignment("center");
2627
- this.SHEET.setFrozenRows(1);
2628
- }
2629
- //----------------------------------------------------------------
2630
- //---- addColumn ---------------------------------------------------
2631
- /**
2632
- * Adding a column to the sheet
2633
- * @param columnName (string) column name
2634
- * @param columnIndex (integer) optional; column index
2635
- */
2636
- addColumn(columnName, columnIndex = 1) {
2637
- const numColumns = this.SHEET.getMaxColumns();
2638
- if (columnIndex <= 0 || columnIndex > numColumns + 1) {
2639
- throw new Error(`Column index ${columnIndex} is out of bounds (1-${numColumns + 1})`);
2640
- }
2641
- if (columnIndex <= numColumns) {
2642
- const headerValue = this.SHEET.getRange(1, columnIndex).getValue();
2643
- if (headerValue !== "") {
2644
- const findFirstEmptyColumn = (startIndex) => {
2645
- let index = startIndex;
2646
- let foundEmpty = false;
2647
- while (index <= numColumns) {
2648
- if (this.SHEET.getRange(1, index).getValue() === "") {
2649
- foundEmpty = true;
2650
- break;
2651
- }
2652
- index++;
2653
- }
2654
- return {
2655
- columnIndex: index,
2656
- foundEmpty
2657
- };
2658
- };
2659
- const result = findFirstEmptyColumn(columnIndex);
2660
- if (!result.foundEmpty) {
2661
- this.SHEET.insertColumnAfter(numColumns);
2662
- columnIndex = numColumns + 1;
2663
- } else {
2664
- this.SHEET.insertColumnBefore(result.columnIndex);
2665
- columnIndex = result.columnIndex;
2666
- }
2667
- }
2668
- } else {
2669
- this.SHEET.insertColumnAfter(numColumns);
2670
- columnIndex = numColumns + 1;
2671
- }
2672
- this.SHEET.getRange(1, columnIndex).setValue(columnName);
2673
- if (this.schema != null && columnName in this.schema && "GoogleSheetsFormat" in this.schema[columnName]) {
2674
- let columnLetter = String.fromCharCode(64 + columnIndex);
2675
- console.log(
2676
- columnName,
2677
- this.schema[columnName]["GoogleSheetsFormat"],
2678
- this.SHEET.getRange(`${columnLetter}:${columnLetter}`).getA1Notation()
2679
- );
2680
- this.SHEET.getRange(`${columnLetter}:${columnLetter}`).setNumberFormat(this.schema[columnName]["GoogleSheetsFormat"]);
2681
- }
2682
- this.columnNames.push(columnName);
2683
- }
2684
- //----------------------------------------------------------------
2685
- //---- formatColumn ------------------------------------------------
2686
- /**
2687
- * Format column as it described in schema
2688
- * @param columnName (string) column name
2689
- */
2690
- formatColumn(columnName) {
2691
- if ("type" in this.schema[columnName]) ;
2692
- }
2693
- //----------------------------------------------------------------
2694
- //---- getColumnIndexByName ----------------------------------------
2695
- /**
2696
- * @param columnName string column name
2697
- * @return integer columnIndex
2698
- */
2699
- getColumnIndexByName(columnName) {
2700
- const columnIndex = this.columnNames.indexOf(columnName);
2701
- if (columnIndex == -1) {
2702
- throw new Error(`Column ${columnName} not found in '${this.SHEET.getName()}' sheet`);
2703
- }
2704
- return columnIndex + 1;
2705
- }
2706
- //----------------------------------------------------------------
2707
- //---- isEmpty -----------------------------------------------------
2708
- /**
2709
- * @return boolean true if sheet is empty, false overwise
2710
- */
2711
- isEmpty() {
2712
- return this.SHEET.getLastRow() === 0 && this.SHEET.getLastColumn() === 0;
2713
- }
2714
- //----------------------------------------------------------------
2715
- //---- areValuesEqual ----------------------------------------------
2716
- /**
2717
- * Comparing to vaariables if they are equal or not
2718
- * @param value1 (mixed)
2719
- * @param value2 (mixed)
2720
- * @return boolean true if equal, falst overwise
2721
- */
2722
- areValuesEqual(value1, value2) {
2723
- var equal = null;
2724
- if (typeof value1 !== "undefined" && typeof value2 !== "undefined" && ((value1 == null ? void 0 : value1.constructor.name) == "Date" || (value2 == null ? void 0 : value2.constructor.name) == "Date")) {
2725
- const normalizeToDate = (value) => {
2726
- if (value === null || value === "") return null;
2727
- if (value.constructor.name == "Date") return value;
2728
- const date = new Date(value);
2729
- return isNaN(date.getTime()) ? null : date;
2730
- };
2731
- const date1 = normalizeToDate(value1);
2732
- const date2 = normalizeToDate(value2);
2733
- if (date1 === null || date2 === null) {
2734
- return value1 === value2;
2735
- }
2736
- equal = date1.getTime() === date2.getTime();
2737
- } else if (typeof value1 == typeof value2) {
2738
- equal = value1 === value2;
2739
- } else if (value1 === void 0 && value2 === "" || value2 === void 0 && value1 === "") {
2740
- equal = true;
2741
- } else if (value1 === null && value2 === "" || value2 === null && value1 === "") {
2742
- equal = true;
2743
- }
2744
- return equal;
2745
- }
2746
- //----------------------------------------------------------------
2747
- //---- areHeadersNeeded ------------------------------------------
2748
- /**
2749
- * Checks if storage is empty and adds headers if needed
2750
- * if destination sheet is empty than header should be created based on unique key columns list
2751
- * @return {boolean} true if headers were added, false if they already existed
2752
- */
2753
- areHeadersNeeded() {
2754
- return this.isEmpty();
2755
- }
2756
- //----------------------------------------------------------------
2757
- };
2758
- const manifest = {
2759
- "name": "GoogleSheetsStorage",
2760
- "description": "Storage for Google Sheets",
2761
- "title": "Google Sheets",
2762
- "version": "0.0.0",
2763
- "author": "OWOX, Inc.",
2764
- "license": "MIT",
2765
- "environment": {
2766
- "node": {
2767
- "enabled": false
2768
- },
2769
- "appscript": {
2770
- "enabled": true
2771
- }
2772
- }
2773
- };
2774
- return {
2775
- GoogleSheetsStorage: GoogleSheetsStorage2,
2776
- manifest
2777
- };
2778
- })();
2779
1797
  const GoogleBigQuery = (function() {
2780
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
1798
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2781
1799
  var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
2782
1800
  //---- constructor -------------------------------------------------
2783
1801
  /**
2784
- * Asbstract class making Google BigQuery updateable in Apps Script
2785
- *
2786
- * @param config (object) instance of AbscractConfig
1802
+ * Abstract class for Google BigQuery storage operations
1803
+ *
1804
+ * @param config (object) instance of AbstractConfig
2787
1805
  * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2788
1806
  * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2789
1807
  * @param description (string) string with storage description }
@@ -2829,22 +1847,29 @@ const GoogleBigQuery = (function() {
2829
1847
  schema,
2830
1848
  description
2831
1849
  );
2832
- this.checkIfGoogleBigQueryIsConnected();
2833
- this.loadTableSchema();
2834
1850
  this.updatedRecordsBuffer = {};
2835
1851
  this.totalRecordsProcessed = 0;
2836
1852
  }
1853
+ //---- init --------------------------------------------------------
1854
+ /**
1855
+ * Initializing storage
1856
+ */
1857
+ async init() {
1858
+ this.checkIfGoogleBigQueryIsConnected();
1859
+ await this.loadTableSchema();
1860
+ }
1861
+ //----------------------------------------------------------------
2837
1862
  //---- loads Google BigQuery Table Schema ---------------------------
2838
- loadTableSchema() {
1863
+ async loadTableSchema() {
2839
1864
  this.existingColumns = this.getAListOfExistingColumns() || {};
2840
1865
  if (Object.keys(this.existingColumns).length == 0) {
2841
- this.createDatasetIfItDoesntExist();
2842
- this.existingColumns = this.createTableIfItDoesntExist();
1866
+ await this.createDatasetIfItDoesntExist();
1867
+ this.existingColumns = await this.createTableIfItDoesntExist();
2843
1868
  } else {
2844
1869
  let selectedFields = this.getSelectedFields();
2845
1870
  let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
2846
1871
  if (newFields.length > 0) {
2847
- this.addNewColumns(newFields);
1872
+ await this.addNewColumns(newFields);
2848
1873
  }
2849
1874
  }
2850
1875
  }
@@ -2882,17 +1907,17 @@ const GoogleBigQuery = (function() {
2882
1907
  return columns;
2883
1908
  }
2884
1909
  //---- createDatasetIfItDoesntExist --------------------------------
2885
- createDatasetIfItDoesntExist() {
1910
+ async createDatasetIfItDoesntExist() {
2886
1911
  let query = `---- Create Dataset if it not exists -----
2887
1912
  `;
2888
1913
  query += `CREATE SCHEMA IF NOT EXISTS \`${this.config.DestinationProjectID.value}.${this.config.DestinationDatasetName.value}\`
2889
1914
  OPTIONS (
2890
1915
  location = '${this.config.DestinationLocation.value}'
2891
1916
  )`;
2892
- this.executeQuery(query);
1917
+ await this.executeQuery(query);
2893
1918
  }
2894
1919
  //---- createTableIfItDoesntExist ----------------------------------
2895
- createTableIfItDoesntExist() {
1920
+ async createTableIfItDoesntExist() {
2896
1921
  let columns = [];
2897
1922
  let columnPartitioned = null;
2898
1923
  let existingColumns = {};
@@ -2928,15 +1953,14 @@ PARTITION BY ${columnPartitioned}`;
2928
1953
  query += `
2929
1954
  OPTIONS(description="${this.description}")`;
2930
1955
  }
2931
- this.executeQuery(query);
1956
+ await this.executeQuery(query);
2932
1957
  this.config.logMessage(`Table ${this.config.DestinationDatasetID.value}.${this.config.DestinationTableName.value} was created`);
2933
1958
  return existingColumns;
2934
1959
  }
2935
1960
  //---- checkIfGoogleBigQueryIsConnected ---------------------
2936
1961
  checkIfGoogleBigQueryIsConnected() {
2937
1962
  if (typeof BigQuery == "undefined") {
2938
- throw new Error(`To import data into Google BigQuery you need to add BigQuery Service first:
2939
- Extension / Apps Script / Editor / Services / + BigQuery API`);
1963
+ throw new Error(`BigQuery client library is not available. Ensure @google-cloud/bigquery is installed.`);
2940
1964
  }
2941
1965
  }
2942
1966
  //---- addNewColumns -----------------------------------------------
@@ -2947,7 +1971,7 @@ OPTIONS(description="${this.description}")`;
2947
1971
  * @param {newColumns} array with a list of new columns
2948
1972
  *
2949
1973
  */
2950
- addNewColumns(newColumns) {
1974
+ async addNewColumns(newColumns) {
2951
1975
  let query = "";
2952
1976
  let columns = [];
2953
1977
  for (var i in newColumns) {
@@ -2969,7 +1993,7 @@ OPTIONS(description="${this.description}")`;
2969
1993
 
2970
1994
  `;
2971
1995
  query += columns.join(",\n");
2972
- this.executeQuery(query);
1996
+ await this.executeQuery(query);
2973
1997
  this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.DestinationDatasetID.value} dataset`);
2974
1998
  }
2975
1999
  }
@@ -2978,17 +2002,17 @@ OPTIONS(description="${this.description}")`;
2978
2002
  * Saving data to a storage
2979
2003
  * @param {data} array of assoc objects with records to save
2980
2004
  */
2981
- saveData(data) {
2982
- data.map((row) => {
2005
+ async saveData(data) {
2006
+ for (const row of data) {
2983
2007
  let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
2984
2008
  if (newFields.length > 0) {
2985
2009
  console.log(newFields);
2986
- this.addNewColumns(newFields);
2010
+ await this.addNewColumns(newFields);
2987
2011
  }
2988
2012
  this.addRecordToBuffer(row);
2989
- this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2990
- });
2991
- this.saveRecordsAddedToBuffer();
2013
+ await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2014
+ }
2015
+ await this.saveRecordsAddedToBuffer();
2992
2016
  }
2993
2017
  // ------- addReordTuBuffer ---------------------
2994
2018
  /**
@@ -3003,24 +2027,24 @@ OPTIONS(description="${this.description}")`;
3003
2027
  * Add records from buffer to a sheet
3004
2028
  * @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
3005
2029
  */
3006
- saveRecordsAddedToBuffer(maxBufferSize = 0) {
2030
+ async saveRecordsAddedToBuffer(maxBufferSize = 0) {
3007
2031
  let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
3008
2032
  if (bufferSize && bufferSize >= maxBufferSize) {
3009
2033
  console.log(`Starting BigQuery MERGE operation for ${bufferSize} records...`);
3010
- this.executeQueryWithSizeLimit();
2034
+ await this.executeQueryWithSizeLimit();
3011
2035
  }
3012
2036
  }
3013
2037
  //---- executeQueryWithSizeLimit ----------------------------------
3014
2038
  /**
3015
2039
  * Executes the MERGE query with automatic size reduction if it exceeds BigQuery limits
3016
2040
  */
3017
- executeQueryWithSizeLimit() {
2041
+ async executeQueryWithSizeLimit() {
3018
2042
  const bufferKeys = Object.keys(this.updatedRecordsBuffer);
3019
2043
  const totalRecords = bufferKeys.length;
3020
2044
  if (totalRecords === 0) {
3021
2045
  return;
3022
2046
  }
3023
- this.executeMergeQueryRecursively(bufferKeys, totalRecords);
2047
+ await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
3024
2048
  this.updatedRecordsBuffer = {};
3025
2049
  }
3026
2050
  //---- executeMergeQueryRecursively --------------------------------
@@ -3029,7 +2053,7 @@ OPTIONS(description="${this.description}")`;
3029
2053
  * @param {Array} recordKeys - Array of record keys to process
3030
2054
  * @param {number} batchSize - Current batch size to attempt
3031
2055
  */
3032
- executeMergeQueryRecursively(recordKeys, batchSize) {
2056
+ async executeMergeQueryRecursively(recordKeys, batchSize) {
3033
2057
  if (recordKeys.length === 0) {
3034
2058
  return;
3035
2059
  }
@@ -3043,11 +2067,11 @@ OPTIONS(description="${this.description}")`;
3043
2067
  const maxQuerySize = 1024 * 1024;
3044
2068
  if (querySize > maxQuerySize) {
3045
2069
  console.log(`Query size (${Math.round(querySize / 1024)}KB) exceeds BigQuery limit. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
3046
- this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
2070
+ await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
3047
2071
  return;
3048
2072
  }
3049
2073
  try {
3050
- this.executeQuery(query);
2074
+ await this.executeQuery(query);
3051
2075
  this.totalRecordsProcessed += currentBatch.length;
3052
2076
  console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
3053
2077
  if (remainingRecords.length > 0) {
@@ -3081,9 +2105,10 @@ OPTIONS(description="${this.description}")`;
3081
2105
  if (record[columnName] === void 0 || record[columnName] === null) {
3082
2106
  columnValue = null;
3083
2107
  } else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
3084
- columnValue = EnvironmentAdapter3.formatDate(record[columnName], "UTC", "yyyy-MM-dd");
2108
+ columnValue = DateUtils3.formatDate(record[columnName]);
3085
2109
  } else if (columnType.toUpperCase() == "DATETIME" && record[columnName] instanceof Date) {
3086
- columnValue = EnvironmentAdapter3.formatDate(record[columnName], "UTC", "yyyy-MM-dd HH:mm:ss");
2110
+ const isoString = record[columnName].toISOString();
2111
+ columnValue = isoString.replace("T", " ").substring(0, 19);
3087
2112
  } else {
3088
2113
  columnValue = this.obfuscateSpecialCharacters(record[columnName]);
3089
2114
  }
@@ -3118,53 +2143,36 @@ OPTIONS(description="${this.description}")`;
3118
2143
  //---- query -------------------------------------------------------
3119
2144
  /**
3120
2145
  * Executes Google BigQuery Query and returns a result
3121
- *
3122
- * @param {query} string
3123
- *
3124
- * @return object
3125
- *
2146
+ *
2147
+ * @param {query} string
2148
+ *
2149
+ * @return Promise<object>
2150
+ *
3126
2151
  */
3127
- executeQuery(query) {
3128
- if (this.config.Environment.value === ENVIRONMENT2.APPS_SCRIPT) {
3129
- return BigQuery.Jobs.query(
3130
- { "query": query, useLegacySql: false },
3131
- this.config.ProjectID.value
3132
- );
3133
- }
3134
- if (this.config.Environment.value === ENVIRONMENT2.NODE) {
3135
- let result = void 0;
3136
- let error = void 0;
3137
- let bigqueryClient = null;
3138
- if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
3139
- const { JWT } = require("google-auth-library");
3140
- const credentials = JSON.parse(this.config.ServiceAccountJson.value);
3141
- const authClient = new JWT({
3142
- email: credentials.client_email,
3143
- key: credentials.private_key,
3144
- scopes: ["https://www.googleapis.com/auth/bigquery"]
3145
- });
3146
- bigqueryClient = new BigQuery({
3147
- projectId: this.config.ProjectID.value || credentials.project_id,
3148
- authClient
3149
- });
3150
- } else {
3151
- throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
3152
- }
3153
- const options = {
3154
- query,
3155
- useLegacySql: false
3156
- };
3157
- bigqueryClient.createQueryJob(options).then(([job]) => job.getQueryResults()).then(([rows]) => rows).then((value) => {
3158
- result = value;
3159
- }).catch((e) => {
3160
- error = e;
2152
+ async executeQuery(query) {
2153
+ let bigqueryClient = null;
2154
+ if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
2155
+ const { JWT } = require("google-auth-library");
2156
+ const credentials = JSON.parse(this.config.ServiceAccountJson.value);
2157
+ const authClient = new JWT({
2158
+ email: credentials.client_email,
2159
+ key: credentials.private_key,
2160
+ scopes: ["https://www.googleapis.com/auth/bigquery"]
3161
2161
  });
3162
- deasync.loopWhile(() => result === void 0 && error === void 0);
3163
- if (error !== void 0) {
3164
- throw error;
3165
- }
3166
- return result;
2162
+ bigqueryClient = new BigQuery({
2163
+ projectId: this.config.ProjectID.value || credentials.project_id,
2164
+ authClient
2165
+ });
2166
+ } else {
2167
+ throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
3167
2168
  }
2169
+ const options = {
2170
+ query,
2171
+ useLegacySql: false
2172
+ };
2173
+ const [job] = await bigqueryClient.createQueryJob(options);
2174
+ const [rows] = await job.getQueryResults();
2175
+ return rows;
3168
2176
  }
3169
2177
  //---- obfuscateSpecialCharacters ----------------------------------
3170
2178
  obfuscateSpecialCharacters(inputString) {
@@ -3260,7 +2268,7 @@ OPTIONS(description="${this.description}")`;
3260
2268
  };
3261
2269
  })();
3262
2270
  const AwsAthena = (function() {
3263
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2271
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
3264
2272
  var AwsAthenaStorage = class AwsAthenaStorage extends AbstractStorage2 {
3265
2273
  //---- constructor -------------------------------------------------
3266
2274
  /**
@@ -3318,9 +2326,21 @@ const AwsAthena = (function() {
3318
2326
  this.initAWS();
3319
2327
  this.updatedRecordsBuffer = {};
3320
2328
  this.existingColumns = {};
3321
- this.setupAthenaDatabase();
3322
2329
  this.uploadSid = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:.]/g, "") + "_" + Math.random().toString(36).substring(2, 15);
3323
2330
  }
2331
+ //---- init --------------------------------------------------------
2332
+ /**
2333
+ * Initializing storage
2334
+ */
2335
+ async init() {
2336
+ const success = await this.setupAthenaDatabase();
2337
+ if (success) {
2338
+ console.log("Database created or already exists");
2339
+ } else {
2340
+ throw new Error("Failed to create database");
2341
+ }
2342
+ }
2343
+ //----------------------------------------------------------------
3324
2344
  //---- initAWS ----------------------------------------------------
3325
2345
  /**
3326
2346
  * Initialize AWS SDK clients
@@ -3352,61 +2372,58 @@ const AwsAthena = (function() {
3352
2372
  /**
3353
2373
  * Create Athena database if it doesn't exist
3354
2374
  */
3355
- createDatabaseIfNotExists() {
2375
+ async createDatabaseIfNotExists() {
3356
2376
  const params = {
3357
2377
  QueryString: `CREATE SCHEMA IF NOT EXISTS \`${this.config.AthenaDatabaseName.value}\``,
3358
2378
  ResultConfiguration: {
3359
2379
  OutputLocation: this.config.AthenaOutputLocation.value
3360
2380
  }
3361
2381
  };
3362
- return this.executeQuery(params, "ddl").then(() => {
3363
- this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
3364
- return true;
3365
- });
2382
+ await this.executeQuery(params, "ddl");
2383
+ this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
2384
+ return true;
3366
2385
  }
3367
2386
  //---- checkTableExists ------------------------------------------
3368
2387
  /**
3369
2388
  * Check if the target table exists in Athena
3370
2389
  */
3371
- checkTableExists() {
2390
+ async checkTableExists() {
3372
2391
  const params = {
3373
2392
  QueryString: `SHOW TABLES IN \`${this.config.AthenaDatabaseName.value}\` LIKE '${this.config.DestinationTableName.value}'`,
3374
2393
  ResultConfiguration: {
3375
2394
  OutputLocation: this.config.AthenaOutputLocation.value
3376
2395
  }
3377
2396
  };
3378
- return this.executeQuery(params, "ddl").then((results) => {
2397
+ try {
2398
+ const results = await this.executeQuery(params, "ddl");
3379
2399
  if (results && results.length > 0) {
3380
- return this.getTableSchema();
2400
+ return await this.getTableSchema();
3381
2401
  }
3382
- return this.createTargetTable();
3383
- }).catch(() => {
3384
- return this.createTargetTable();
3385
- });
2402
+ return await this.createTargetTable();
2403
+ } catch {
2404
+ return await this.createTargetTable();
2405
+ }
3386
2406
  }
3387
2407
  //---- getTableSchema -------------------------------------------
3388
2408
  /**
3389
2409
  * Get the schema of the existing table
3390
2410
  */
3391
- getTableSchema() {
2411
+ async getTableSchema() {
3392
2412
  const params = {
3393
2413
  QueryString: `SHOW COLUMNS IN \`${this.config.AthenaDatabaseName.value}\`.\`${this.config.DestinationTableName.value}\``,
3394
2414
  ResultConfiguration: {
3395
2415
  OutputLocation: this.config.AthenaOutputLocation.value
3396
2416
  }
3397
2417
  };
3398
- return this.executeQuery(params, "ddl").then((results) => {
3399
- let columns = {};
3400
- if (results && results.length > 0) {
3401
- results.forEach((row) => {
3402
- columns[row] = this.getColumnType(row);
3403
- });
3404
- }
3405
- this.existingColumns = columns;
3406
- return columns;
3407
- }).catch((error) => {
3408
- return {};
3409
- });
2418
+ const results = await this.executeQuery(params, "ddl");
2419
+ let columns = {};
2420
+ if (results && results.length > 0) {
2421
+ results.forEach((row) => {
2422
+ columns[row] = this.getColumnType(row);
2423
+ });
2424
+ }
2425
+ this.existingColumns = columns;
2426
+ return columns;
3410
2427
  }
3411
2428
  //---- createTargetTable ----------------------------------------------
3412
2429
  /**
@@ -3496,45 +2513,42 @@ const AwsAthena = (function() {
3496
2513
  * @param {Array} data - Array of objects with records to save
3497
2514
  * @returns {Promise}
3498
2515
  */
3499
- saveData(data) {
3500
- let done = false;
3501
- this.checkTableExists().then(() => {
3502
- const allColumns = /* @__PURE__ */ new Set();
3503
- if (data.length > 0) {
3504
- data.forEach((row) => {
3505
- Object.keys(row).forEach((column) => allColumns.add(column));
3506
- });
3507
- }
3508
- if (this.config.Fields.value) {
3509
- this.getSelectedFields().forEach((columnName) => {
3510
- if (columnName && !allColumns.has(columnName)) {
3511
- allColumns.add(columnName);
3512
- if (data.length > 0) {
3513
- data.forEach((row) => {
3514
- if (!row[columnName]) {
3515
- row[columnName] = "";
3516
- }
3517
- });
3518
- }
2516
+ async saveData(data) {
2517
+ await this.checkTableExists();
2518
+ const allColumns = /* @__PURE__ */ new Set();
2519
+ if (data.length > 0) {
2520
+ data.forEach((row) => {
2521
+ Object.keys(row).forEach((column) => allColumns.add(column));
2522
+ });
2523
+ }
2524
+ if (this.config.Fields.value) {
2525
+ this.getSelectedFields().forEach((columnName) => {
2526
+ if (columnName && !allColumns.has(columnName)) {
2527
+ allColumns.add(columnName);
2528
+ if (data.length > 0) {
2529
+ data.forEach((row) => {
2530
+ if (!row[columnName]) {
2531
+ row[columnName] = "";
2532
+ }
2533
+ });
3519
2534
  }
3520
- });
3521
- }
3522
- const existingColumnsSet = new Set(Object.keys(this.existingColumns));
3523
- const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
3524
- if (newColumns.length > 0) {
3525
- return this.addNewColumns(newColumns);
3526
- }
3527
- return Promise.resolve();
3528
- }).then(() => {
3529
- if (data.length === 0) {
3530
- done = true;
3531
- return;
3532
- }
3533
- this.config.logMessage(`Saving ${data.length} records to Athena`);
3534
- const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
3535
- return this.uploadDataToS3TempFolder(data, tempFolder).then(() => this.createTempTable(tempFolder, this.uploadSid)).then((tempTableName) => this.mergeDataFromTempTable(tempTableName, this.uploadSid)).then((tempTableName) => this.cleanupTempResources(tempFolder, tempTableName)).then(() => done = true);
3536
- });
3537
- deasync.loopWhile(() => !done);
2535
+ }
2536
+ });
2537
+ }
2538
+ const existingColumnsSet = new Set(Object.keys(this.existingColumns));
2539
+ const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
2540
+ if (newColumns.length > 0) {
2541
+ await this.addNewColumns(newColumns);
2542
+ }
2543
+ if (data.length === 0) {
2544
+ return;
2545
+ }
2546
+ this.config.logMessage(`Saving ${data.length} records to Athena`);
2547
+ const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
2548
+ await this.uploadDataToS3TempFolder(data, tempFolder);
2549
+ const tempTableName = await this.createTempTable(tempFolder, this.uploadSid);
2550
+ await this.mergeDataFromTempTable(tempTableName, this.uploadSid);
2551
+ await this.cleanupTempResources(tempFolder, tempTableName);
3538
2552
  }
3539
2553
  //---- uploadDataToS3TempFolder ---------------------------------
3540
2554
  /**
@@ -3723,6 +2737,7 @@ const AwsAthena = (function() {
3723
2737
  QueryExecutionId: queryExecutionId
3724
2738
  };
3725
2739
  return this.athenaClient.send(new GetQueryExecutionCommand(params)).then((data) => {
2740
+ var _a;
3726
2741
  const state = data.QueryExecution.Status.State;
3727
2742
  if (state === "SUCCEEDED") {
3728
2743
  return new Promise((resolve) => {
@@ -3731,6 +2746,7 @@ const AwsAthena = (function() {
3731
2746
  }, 3e3);
3732
2747
  });
3733
2748
  } else if (state === "FAILED" || state === "CANCELLED") {
2749
+ this.config.logMessage(`Query ${queryExecutionId} ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}. Error: ${((_a = data.QueryExecution.Status.Error) == null ? void 0 : _a.Message) || ""}`);
3734
2750
  throw new Error(`Query ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}`);
3735
2751
  } else {
3736
2752
  return new Promise((resolve) => {
@@ -3890,12 +2906,11 @@ const AwsAthena = (function() {
3890
2906
  };
3891
2907
  })();
3892
2908
  const Storages = {
3893
- GoogleSheets,
3894
2909
  GoogleBigQuery,
3895
2910
  AwsAthena
3896
2911
  };
3897
2912
  const XAds = (function() {
3898
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2913
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
3899
2914
  const XAdsHelper = {
3900
2915
  /**
3901
2916
  * Parse fields string into a structured object
@@ -4789,34 +3804,34 @@ const XAds = (function() {
4789
3804
  * @param {string} [opts.end_time]
4790
3805
  * @returns {Array<Object>}
4791
3806
  */
4792
- fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
4793
- EnvironmentAdapter3.sleep(this.config.AdsApiDelay.value * 1e3);
3807
+ async fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
3808
+ await AsyncUtils3.delay(this.config.AdsApiDelay.value * 1e3);
4794
3809
  switch (nodeName) {
4795
3810
  case "accounts": {
4796
- const resp = this._getData(`accounts/${accountId}`, "accounts", fields);
3811
+ const resp = await this._getData(`accounts/${accountId}`, "accounts", fields);
4797
3812
  return [resp.data];
4798
3813
  }
4799
3814
  case "campaigns":
4800
3815
  case "line_items":
4801
3816
  case "promoted_tweets":
4802
3817
  case "tweets":
4803
- return this._catalogFetch({
3818
+ return await this._catalogFetch({
4804
3819
  nodeName,
4805
3820
  accountId,
4806
3821
  fields,
4807
3822
  pageSize: this.config.DataMaxCount.value
4808
3823
  });
4809
3824
  case "cards":
4810
- return this._catalogFetch({
3825
+ return await this._catalogFetch({
4811
3826
  nodeName,
4812
3827
  accountId,
4813
3828
  fields,
4814
3829
  pageSize: this.config.CardsMaxCountPerRequest.value
4815
3830
  });
4816
3831
  case "cards_all":
4817
- return this._fetchAllCards(accountId, fields);
3832
+ return await this._fetchAllCards(accountId, fields);
4818
3833
  case "stats":
4819
- return this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
3834
+ return await this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
4820
3835
  default:
4821
3836
  throw new ConfigurationError(`Unknown node: ${nodeName}`);
4822
3837
  }
@@ -4852,7 +3867,7 @@ const XAds = (function() {
4852
3867
  /**
4853
3868
  * Shared logic for non-time-series endpoints
4854
3869
  */
4855
- _catalogFetch({ nodeName, accountId, fields, pageSize }) {
3870
+ async _catalogFetch({ nodeName, accountId, fields, pageSize }) {
4856
3871
  const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
4857
3872
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
4858
3873
  if (missingKeys.length > 0) {
@@ -4876,7 +3891,7 @@ const XAds = (function() {
4876
3891
  console.log("deleting cached tweets");
4877
3892
  this._tweetsCache.delete(accountId);
4878
3893
  }
4879
- let all = this._fetchPages({
3894
+ let all = await this._fetchPages({
4880
3895
  accountId,
4881
3896
  nodeName,
4882
3897
  fields,
@@ -4900,7 +3915,7 @@ const XAds = (function() {
4900
3915
  /**
4901
3916
  * Shared pagination logic
4902
3917
  */
4903
- _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
3918
+ async _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
4904
3919
  const all = [];
4905
3920
  let cursor = null;
4906
3921
  const MAX_PAGES = 100;
@@ -4911,7 +3926,7 @@ const XAds = (function() {
4911
3926
  ...extraParams,
4912
3927
  ...cursor ? { cursor } : {}
4913
3928
  };
4914
- const resp = this._getData(
3929
+ const resp = await this._getData(
4915
3930
  `accounts/${accountId}/${nodeName}`,
4916
3931
  nodeName,
4917
3932
  fields,
@@ -4932,15 +3947,15 @@ const XAds = (function() {
4932
3947
  * Fetch all cards by first collecting URIs from tweets,
4933
3948
  * then calling the cards/all endpoint in chunks.
4934
3949
  */
4935
- _fetchAllCards(accountId, fields) {
4936
- const tweets = this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
3950
+ async _fetchAllCards(accountId, fields) {
3951
+ const tweets = await this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
4937
3952
  const uris = tweets.map((t) => t.card_uri).filter(Boolean);
4938
3953
  if (!uris.length) return [];
4939
3954
  const all = [];
4940
3955
  const chunkSize = this.config.CardsMaxCountPerRequest.value;
4941
3956
  for (let i = 0; i < uris.length; i += chunkSize) {
4942
3957
  const chunk = uris.slice(i, i + chunkSize);
4943
- const resp = this._getData(
3958
+ const resp = await this._getData(
4944
3959
  `accounts/${accountId}/cards/all`,
4945
3960
  "cards_all",
4946
3961
  fields,
@@ -4957,18 +3972,18 @@ const XAds = (function() {
4957
3972
  /**
4958
3973
  * Stats are time-series and need flattening of `metrics`
4959
3974
  */
4960
- _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
3975
+ async _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
4961
3976
  const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
4962
3977
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
4963
3978
  if (missingKeys.length > 0) {
4964
3979
  throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
4965
3980
  }
4966
- const promos = this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
3981
+ const promos = await this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
4967
3982
  const ids = promos.map((r) => r.id);
4968
3983
  if (!ids.length) return [];
4969
3984
  const e = new Date(end_time);
4970
3985
  e.setDate(e.getDate() + 1);
4971
- const endStr = EnvironmentAdapter3.formatDate(e, "UTC", "yyyy-MM-dd");
3986
+ const endStr = DateUtils3.formatDate(e);
4972
3987
  const result = [];
4973
3988
  for (let i = 0; i < ids.length; i += this.config.StatsMaxEntityIds.value) {
4974
3989
  const batch = ids.slice(i, i + this.config.StatsMaxEntityIds.value).join(",");
@@ -4981,7 +3996,7 @@ const XAds = (function() {
4981
3996
  end_time: endStr
4982
3997
  };
4983
3998
  for (const placement of ["ALL_ON_TWITTER", "PUBLISHER_NETWORK"]) {
4984
- const raw = this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
3999
+ const raw = await this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
4985
4000
  const arr = Array.isArray(raw.data) ? raw.data : [raw.data];
4986
4001
  arr.forEach((h) => {
4987
4002
  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
@@ -5017,18 +4032,19 @@ const XAds = (function() {
5017
4032
  /**
5018
4033
  * Pull JSON from the Ads API (raw, no field-filter).
5019
4034
  */
5020
- _rawFetch(path, params = {}) {
4035
+ async _rawFetch(path, params = {}) {
5021
4036
  const url = `${this.BASE_URL}${this.config.Version.value}/${path}`;
5022
4037
  const qs = Object.keys(params).length ? "?" + Object.entries(params).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
5023
4038
  const finalUrl = url + qs;
5024
4039
  const oauth = this._generateOAuthHeader({ method: "GET", url, params });
5025
- EnvironmentAdapter3.sleep(1e3);
5026
- const resp = this.urlFetchWithRetry(finalUrl, {
4040
+ await AsyncUtils3.delay(1e3);
4041
+ const resp = await this.urlFetchWithRetry(finalUrl, {
5027
4042
  method: "GET",
5028
4043
  headers: { Authorization: oauth, "Content-Type": "application/json" },
5029
4044
  muteHttpExceptions: true
5030
4045
  });
5031
- return JSON.parse(resp.getContentText());
4046
+ const text = await resp.getContentText();
4047
+ return JSON.parse(text);
5032
4048
  }
5033
4049
  /**
5034
4050
  * Determines if a X Ads API error is valid for retry
@@ -5051,8 +4067,8 @@ const XAds = (function() {
5051
4067
  }
5052
4068
  return false;
5053
4069
  }
5054
- _getData(path, nodeName, fields, extraParams = {}) {
5055
- const json = this._rawFetch(path, extraParams);
4070
+ async _getData(path, nodeName, fields, extraParams = {}) {
4071
+ const json = await this._rawFetch(path, extraParams);
5056
4072
  if (!json.data) return json;
5057
4073
  const arr = Array.isArray(json.data) ? json.data : [json.data];
5058
4074
  const filtered = this._filterBySchema(arr, nodeName, fields);
@@ -5091,7 +4107,7 @@ const XAds = (function() {
5091
4107
  const { ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret } = this.config;
5092
4108
  const oauth = {
5093
4109
  oauth_consumer_key: ConsumerKey.value,
5094
- oauth_nonce: EnvironmentAdapter3.getUuid().replace(/-/g, ""),
4110
+ oauth_nonce: CryptoUtils3.getUuid().replace(/-/g, ""),
5095
4111
  oauth_signature_method: "HMAC-SHA1",
5096
4112
  oauth_timestamp: Math.floor(Date.now() / 1e3),
5097
4113
  oauth_token: AccessToken.value,
@@ -5106,9 +4122,9 @@ const XAds = (function() {
5106
4122
  )
5107
4123
  ].join("&");
5108
4124
  const signingKey = encodeURIComponent(ConsumerSecret.value) + "&" + encodeURIComponent(AccessTokenSecret.value);
5109
- oauth.oauth_signature = EnvironmentAdapter3.base64Encode(
5110
- EnvironmentAdapter3.computeHmacSignature(
5111
- EnvironmentAdapter3.MacAlgorithm.HMAC_SHA_1,
4125
+ oauth.oauth_signature = CryptoUtils3.base64Encode(
4126
+ CryptoUtils3.computeHmacSignature(
4127
+ CryptoUtils3.MacAlgorithm.HMAC_SHA_1,
5112
4128
  baseString,
5113
4129
  signingKey
5114
4130
  )
@@ -5124,7 +4140,7 @@ const XAds = (function() {
5124
4140
  }
5125
4141
  };
5126
4142
  var XAdsConnector = class XAdsConnector extends AbstractConnector3 {
5127
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
4143
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
5128
4144
  super(config, source, null, runConfig);
5129
4145
  this.storageName = storageName;
5130
4146
  }
@@ -5132,12 +4148,12 @@ const XAds = (function() {
5132
4148
  * Main method - entry point for the import process
5133
4149
  * Processes all nodes defined in the fields configuration
5134
4150
  */
5135
- startImportProcess() {
4151
+ async startImportProcess() {
5136
4152
  const fields = XAdsHelper.parseFields(this.config.Fields.value);
5137
4153
  const accountIds = XAdsHelper.parseAccountIds(this.config.AccountIDs.value);
5138
4154
  for (const accountId of accountIds) {
5139
4155
  for (const nodeName in fields) {
5140
- this.processNode({
4156
+ await this.processNode({
5141
4157
  nodeName,
5142
4158
  accountId,
5143
4159
  fields: fields[nodeName] || []
@@ -5153,15 +4169,15 @@ const XAds = (function() {
5153
4169
  * @param {string} options.accountId - Account ID
5154
4170
  * @param {Array<string>} options.fields - Array of fields to fetch
5155
4171
  */
5156
- processNode({ nodeName, accountId, fields }) {
4172
+ async processNode({ nodeName, accountId, fields }) {
5157
4173
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
5158
- this.processTimeSeriesNode({
4174
+ await this.processTimeSeriesNode({
5159
4175
  nodeName,
5160
4176
  accountId,
5161
4177
  fields
5162
4178
  });
5163
4179
  } else {
5164
- this.processCatalogNode({
4180
+ await this.processCatalogNode({
5165
4181
  nodeName,
5166
4182
  accountId,
5167
4183
  fields
@@ -5176,7 +4192,7 @@ const XAds = (function() {
5176
4192
  * @param {Array<string>} options.fields - Array of fields to fetch
5177
4193
  * @param {Object} options.storage - Storage instance
5178
4194
  */
5179
- processTimeSeriesNode({ nodeName, accountId, fields }) {
4195
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
5180
4196
  var _a;
5181
4197
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
5182
4198
  if (daysToFetch <= 0) {
@@ -5186,12 +4202,13 @@ const XAds = (function() {
5186
4202
  for (let i = 0; i < daysToFetch; i++) {
5187
4203
  const currentDate = new Date(startDate);
5188
4204
  currentDate.setDate(currentDate.getDate() + i);
5189
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
5190
- const data = this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
4205
+ const formattedDate = DateUtils3.formatDate(currentDate);
4206
+ const data = await this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
5191
4207
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
5192
4208
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
5193
4209
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
5194
- this.getStorageByNode(nodeName).saveData(preparedData);
4210
+ const storage = await this.getStorageByNode(nodeName);
4211
+ await storage.saveData(preparedData);
5195
4212
  }
5196
4213
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
5197
4214
  this.config.updateLastRequstedDate(currentDate);
@@ -5206,13 +4223,14 @@ const XAds = (function() {
5206
4223
  * @param {Array<string>} options.fields - Array of fields to fetch
5207
4224
  * @param {Object} options.storage - Storage instance
5208
4225
  */
5209
- processCatalogNode({ nodeName, accountId, fields }) {
4226
+ async processCatalogNode({ nodeName, accountId, fields }) {
5210
4227
  var _a;
5211
- const data = this.source.fetchData({ nodeName, accountId, fields });
4228
+ const data = await this.source.fetchData({ nodeName, accountId, fields });
5212
4229
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
5213
4230
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
5214
4231
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
5215
- this.getStorageByNode(nodeName).saveData(preparedData);
4232
+ const storage = await this.getStorageByNode(nodeName);
4233
+ await storage.saveData(preparedData);
5216
4234
  }
5217
4235
  }
5218
4236
  /**
@@ -5220,7 +4238,7 @@ const XAds = (function() {
5220
4238
  * @param {string} nodeName - Name of the node
5221
4239
  * @returns {Object} Storage instance
5222
4240
  */
5223
- getStorageByNode(nodeName) {
4241
+ async getStorageByNode(nodeName) {
5224
4242
  if (!("storages" in this)) {
5225
4243
  this.storages = {};
5226
4244
  }
@@ -5238,6 +4256,7 @@ const XAds = (function() {
5238
4256
  this.source.fieldsSchema[nodeName].fields,
5239
4257
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
5240
4258
  );
4259
+ await this.storages[nodeName].init();
5241
4260
  }
5242
4261
  return this.storages[nodeName];
5243
4262
  }
@@ -5254,7 +4273,7 @@ const XAds = (function() {
5254
4273
  };
5255
4274
  })();
5256
4275
  const TikTokAds = (function() {
5257
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
4276
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
5258
4277
  class TiktokMarketingApiProvider {
5259
4278
  constructor(appId, accessToken, appSecret, isSandbox = false) {
5260
4279
  this.BASE_URL = "https://business-api.tiktok.com/open_api/";
@@ -5278,7 +4297,7 @@ const TikTokAds = (function() {
5278
4297
  getApiVersion() {
5279
4298
  return this.API_VERSION;
5280
4299
  }
5281
- makeRequest(options) {
4300
+ async makeRequest(options) {
5282
4301
  const { url, method, data } = options;
5283
4302
  const headers = {
5284
4303
  "Access-Token": this.accessToken,
@@ -5287,21 +4306,21 @@ const TikTokAds = (function() {
5287
4306
  let backoff = this.INITIAL_BACKOFF;
5288
4307
  for (let retries = 0; retries < this.MAX_RETRIES; retries++) {
5289
4308
  try {
5290
- const response = EnvironmentAdapter3.fetch(url, {
4309
+ const response = await HttpUtils3.fetch(url, {
5291
4310
  method,
5292
4311
  headers,
5293
- body: data ? JSON.stringify(data) : null,
5294
- muteHttpExceptions: true
4312
+ body: data ? JSON.stringify(data) : null
5295
4313
  });
5296
4314
  const responseCode = response.getResponseCode();
4315
+ const text = await response.getContentText();
5297
4316
  if (responseCode !== this.SUCCESS_RESPONSE_CODE) {
5298
- throw new Error(`TikTok API error: ${response.getContentText()}`);
4317
+ throw new Error(`TikTok API error: ${text}`);
5299
4318
  }
5300
- const jsonData = JSON.parse(response.getContentText());
4319
+ const jsonData = JSON.parse(text);
5301
4320
  if (jsonData.code !== this.SUCCESS_CODE) {
5302
4321
  if (jsonData.code === this.RATE_LIMIT_CODE) {
5303
4322
  console.error("TikTok Marketing API rate limit exceeded. Retrying...");
5304
- EnvironmentAdapter3.sleep(backoff);
4323
+ await AsyncUtils3.delay(backoff);
5305
4324
  backoff *= 2;
5306
4325
  continue;
5307
4326
  }
@@ -5310,7 +4329,7 @@ const TikTokAds = (function() {
5310
4329
  return jsonData;
5311
4330
  } catch (error) {
5312
4331
  if (retries < this.MAX_RETRIES - 1 && error.message.includes("rate limit")) {
5313
- EnvironmentAdapter3.sleep(backoff);
4332
+ await AsyncUtils3.delay(backoff);
5314
4333
  backoff *= 2;
5315
4334
  } else {
5316
4335
  throw error;
@@ -5318,7 +4337,7 @@ const TikTokAds = (function() {
5318
4337
  }
5319
4338
  }
5320
4339
  }
5321
- handlePagination(endpoint, params = {}) {
4340
+ async handlePagination(endpoint, params = {}) {
5322
4341
  let allData = [];
5323
4342
  let page = 1;
5324
4343
  let hasMorePages = true;
@@ -5326,7 +4345,7 @@ const TikTokAds = (function() {
5326
4345
  while (hasMorePages) {
5327
4346
  const paginatedParams = { ...params, page, page_size: pageSize };
5328
4347
  const url = this.buildUrl(endpoint, paginatedParams);
5329
- const response = this.makeRequest({ url, method: "GET" });
4348
+ const response = await this.makeRequest({ url, method: "GET" });
5330
4349
  const pageData = response.data.list || [];
5331
4350
  allData = allData.concat(pageData);
5332
4351
  const total = response.data.page_info ? response.data.page_info.total_number : 0;
@@ -5334,7 +4353,7 @@ const TikTokAds = (function() {
5334
4353
  hasMorePages = currentCount < total && pageData.length > 0;
5335
4354
  page++;
5336
4355
  if (hasMorePages) {
5337
- EnvironmentAdapter3.sleep(100);
4356
+ await AsyncUtils3.delay(100);
5338
4357
  }
5339
4358
  }
5340
4359
  return allData;
@@ -6417,7 +5436,7 @@ const TikTokAds = (function() {
6417
5436
  * @param {Date} endDate - End date for time-series data (optional)
6418
5437
  * @return {array} - Array of data objects
6419
5438
  */
6420
- fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
5439
+ async fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
6421
5440
  if (!this.fieldsSchema[nodeName]) {
6422
5441
  throw new Error(`Unknown node type: ${nodeName}`);
6423
5442
  }
@@ -6438,8 +5457,8 @@ const TikTokAds = (function() {
6438
5457
  let formattedStartDate = null;
6439
5458
  let formattedEndDate = null;
6440
5459
  if (startDate) {
6441
- formattedStartDate = EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd");
6442
- formattedEndDate = endDate ? EnvironmentAdapter3.formatDate(endDate, "UTC", "yyyy-MM-dd") : formattedStartDate;
5460
+ formattedStartDate = DateUtils3.formatDate(startDate);
5461
+ formattedEndDate = endDate ? DateUtils3.formatDate(endDate) : formattedStartDate;
6443
5462
  }
6444
5463
  let filtering = null;
6445
5464
  if (this.config.IncludeDeleted && this.config.IncludeDeleted.value) {
@@ -6661,11 +5680,11 @@ const TikTokAds = (function() {
6661
5680
  }
6662
5681
  };
6663
5682
  var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector3 {
6664
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
5683
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
6665
5684
  super(config, source, null, runConfig);
6666
5685
  this.storageName = storageName;
6667
5686
  }
6668
- startImportProcess() {
5687
+ async startImportProcess() {
6669
5688
  try {
6670
5689
  let advertiserIds = TikTokAdsHelper.parseAdvertiserIds(this.config.AdvertiserIDs.value || "");
6671
5690
  if (!advertiserIds || advertiserIds.length === 0) {
@@ -6689,7 +5708,7 @@ const TikTokAds = (function() {
6689
5708
  }
6690
5709
  }
6691
5710
  if (Object.keys(catalogNodes).length > 0) {
6692
- this.importCatalogData(catalogNodes, advertiserIds);
5711
+ await this.importCatalogData(catalogNodes, advertiserIds);
6693
5712
  }
6694
5713
  if (Object.keys(timeSeriesNodes).length > 0) {
6695
5714
  try {
@@ -6698,7 +5717,7 @@ const TikTokAds = (function() {
6698
5717
  this.config.logMessage("There is nothing to import in this data range");
6699
5718
  return;
6700
5719
  }
6701
- this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
5720
+ await this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
6702
5721
  } catch (error) {
6703
5722
  this.config.logMessage(`Error determining date range: ${error.message}`);
6704
5723
  console.error(error.stack);
@@ -6717,35 +5736,36 @@ const TikTokAds = (function() {
6717
5736
  }
6718
5737
  /**
6719
5738
  * Imports all catalog (non-time-series) data types
6720
- *
5739
+ *
6721
5740
  * @param {object} catalogNodes - Object with node names as keys and field arrays as values
6722
5741
  * @param {array} advertiserIds - List of advertiser IDs to fetch data for
6723
5742
  */
6724
- importCatalogData(catalogNodes, advertiserIds) {
5743
+ async importCatalogData(catalogNodes, advertiserIds) {
6725
5744
  for (var nodeName in catalogNodes) {
6726
5745
  this.config.logMessage(`Starting import for ${nodeName} data...`);
6727
- this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
5746
+ await this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
6728
5747
  }
6729
5748
  }
6730
5749
  /**
6731
5750
  * Imports catalog (not time series) data
6732
- *
5751
+ *
6733
5752
  * @param {string} nodeName - Node name
6734
5753
  * @param {array} advertiserIds - List of advertiser IDs
6735
5754
  * @param {array} fields - List of fields
6736
5755
  */
6737
- startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
5756
+ async startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
6738
5757
  var _a;
6739
5758
  this.config.logMessage(`Fetching all available fields for ${nodeName}`);
6740
5759
  for (var i in advertiserIds) {
6741
5760
  let advertiserId = advertiserIds[i];
6742
5761
  try {
6743
- let data = this.source.fetchData(nodeName, advertiserId, fields);
5762
+ let data = await this.source.fetchData(nodeName, advertiserId, fields);
6744
5763
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for advertiser ${advertiserId}` : `No records have been fetched`);
6745
5764
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
6746
5765
  try {
6747
5766
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
6748
- this.getStorageByNode(nodeName).saveData(preparedData);
5767
+ const storage = await this.getStorageByNode(nodeName);
5768
+ await storage.saveData(preparedData);
6749
5769
  } catch (storageError) {
6750
5770
  this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
6751
5771
  console.error(`Error details: ${storageError.stack}`);
@@ -6759,29 +5779,30 @@ const TikTokAds = (function() {
6759
5779
  }
6760
5780
  /**
6761
5781
  * Imports time series data
6762
- *
5782
+ *
6763
5783
  * @param {array} advertiserIds - List of advertiser IDs
6764
5784
  * @param {object} timeSeriesNodes - Object of properties, each is array of fields
6765
5785
  * @param {Date} startDate - Start date
6766
5786
  * @param {number} daysToFetch - Number of days to fetch
6767
5787
  */
6768
- startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
5788
+ async startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
6769
5789
  var _a;
6770
5790
  for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
6771
5791
  const currentDate = new Date(startDate);
6772
5792
  currentDate.setDate(currentDate.getDate() + daysShift);
6773
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
5793
+ const formattedDate = DateUtils3.formatDate(currentDate);
6774
5794
  this.config.logMessage(`Processing data for date: ${formattedDate}`);
6775
5795
  for (let advertiserId of advertiserIds) {
6776
5796
  for (var nodeName in timeSeriesNodes) {
6777
5797
  try {
6778
5798
  this.config.logMessage(`Start importing data for ${formattedDate}: ${advertiserId}/${nodeName}`);
6779
- let data = this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
5799
+ let data = await this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
6780
5800
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
6781
5801
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
6782
5802
  try {
6783
5803
  const preparedData = data.length ? this.addMissingFieldsToData(data, timeSeriesNodes[nodeName]) : data;
6784
- this.getStorageByNode(nodeName).saveData(preparedData);
5804
+ const storage = await this.getStorageByNode(nodeName);
5805
+ await storage.saveData(preparedData);
6785
5806
  } catch (storageError) {
6786
5807
  this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
6787
5808
  console.error(`Error details: ${storageError.stack}`);
@@ -6805,7 +5826,7 @@ const TikTokAds = (function() {
6805
5826
  * @param {array} requestedFields - List of requested fields
6806
5827
  * @return {AbstractStorage} - Storage instance
6807
5828
  */
6808
- getStorageByNode(nodeName) {
5829
+ async getStorageByNode(nodeName) {
6809
5830
  if (!("storages" in this)) {
6810
5831
  this.storages = {};
6811
5832
  }
@@ -6827,6 +5848,7 @@ const TikTokAds = (function() {
6827
5848
  this.source.fieldsSchema[nodeName]["fields"] || {},
6828
5849
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
6829
5850
  );
5851
+ await this.storages[nodeName].init();
6830
5852
  }
6831
5853
  return this.storages[nodeName];
6832
5854
  }
@@ -6844,35 +5866,9 @@ const TikTokAds = (function() {
6844
5866
  this.config.logMessage(`Cleaning up data older than ${keepDays} days...`);
6845
5867
  const cutoffDate = /* @__PURE__ */ new Date();
6846
5868
  cutoffDate.setDate(cutoffDate.getDate() - keepDays);
6847
- const formattedCutoffDate = EnvironmentAdapter3.formatDate(cutoffDate, "UTC", "yyyy-MM-dd");
5869
+ DateUtils3.formatDate(cutoffDate);
6848
5870
  for (var nodeName in this.source.fieldsSchema) {
6849
- if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) {
6850
- try {
6851
- const storage = this.getStorageByNode(nodeName);
6852
- if (storage instanceof GoogleSheetsStorage) {
6853
- const dateField = this.source.fieldsSchema[nodeName]["fields"]["date_start"] ? "date_start" : "stat_time_day";
6854
- let keysToDelete = [];
6855
- for (const uniqueKey in storage.values) {
6856
- const record = storage.values[uniqueKey];
6857
- const rowDate = record[dateField];
6858
- if (rowDate && rowDate < cutoffDate) {
6859
- keysToDelete.push(uniqueKey);
6860
- }
6861
- }
6862
- let deletedCount = 0;
6863
- for (const key of keysToDelete) {
6864
- storage.deleteRecord(key);
6865
- deletedCount++;
6866
- }
6867
- if (deletedCount > 0) {
6868
- this.config.logMessage(`Deleted ${deletedCount} rows from ${nodeName} that were older than ${formattedCutoffDate}`);
6869
- }
6870
- }
6871
- } catch (error) {
6872
- this.config.logMessage(`Error cleaning up old data from ${nodeName}: ${error.message}`);
6873
- console.error(`Error details: ${error.stack}`);
6874
- }
6875
- }
5871
+ if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) ;
6876
5872
  }
6877
5873
  }
6878
5874
  };
@@ -6889,7 +5885,7 @@ const TikTokAds = (function() {
6889
5885
  };
6890
5886
  })();
6891
5887
  const RedditAds = (function() {
6892
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
5888
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
6893
5889
  const RedditAdsHelper = {
6894
5890
  /**
6895
5891
  * Parse fields string into a structured object
@@ -9925,7 +8921,7 @@ const RedditAds = (function() {
9925
8921
  * @param {Date|null} startDate - The start date for data fetching.
9926
8922
  * @returns {Array} An array of data records.
9927
8923
  */
9928
- fetchData(nodeName, accountId, fields, startDate = null) {
8924
+ async fetchData(nodeName, accountId, fields, startDate = null) {
9929
8925
  var _a;
9930
8926
  console.log(`Fetching data from ${nodeName}/${accountId} for ${startDate}`);
9931
8927
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -9933,7 +8929,7 @@ const RedditAds = (function() {
9933
8929
  if (missingKeys.length > 0) {
9934
8930
  throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
9935
8931
  }
9936
- const tokenResponse = this.getRedditAccessToken(
8932
+ const tokenResponse = await this.getRedditAccessToken(
9937
8933
  this.config.ClientId.value,
9938
8934
  this.config.ClientSecret.value,
9939
8935
  this.config.RedirectUri.value,
@@ -9943,7 +8939,7 @@ const RedditAds = (function() {
9943
8939
  this.config.AccessToken.value = tokenResponse.accessToken;
9944
8940
  }
9945
8941
  const baseUrl = "https://ads-api.reddit.com/api/v3/";
9946
- let formattedDate = startDate ? EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd") : null;
8942
+ let formattedDate = startDate ? DateUtils3.formatDate(startDate) : null;
9947
8943
  let headers = {
9948
8944
  "Accept": "application/json",
9949
8945
  "User-Agent": this.config.UserAgent.value,
@@ -9969,8 +8965,9 @@ const RedditAds = (function() {
9969
8965
  let nextPageURL = finalUrl;
9970
8966
  while (nextPageURL) {
9971
8967
  try {
9972
- const response = this.urlFetchWithRetry(nextPageURL, options);
9973
- const jsonData = JSON.parse(response.getContentText());
8968
+ const response = await this.urlFetchWithRetry(nextPageURL, options);
8969
+ const text = await response.getContentText();
8970
+ const jsonData = JSON.parse(text);
9974
8971
  if ("data" in jsonData) {
9975
8972
  nextPageURL = jsonData.pagination ? jsonData.pagination.next_url : null;
9976
8973
  if (jsonData && jsonData.data && jsonData.data.metrics) {
@@ -9987,7 +8984,7 @@ const RedditAds = (function() {
9987
8984
  }
9988
8985
  } catch (error) {
9989
8986
  if (error.statusCode === HTTP_STATUS2.UNAUTHORIZED) {
9990
- const newTokenResponse = this.getRedditAccessToken(
8987
+ const newTokenResponse = await this.getRedditAccessToken(
9991
8988
  this.config.ClientId.value,
9992
8989
  this.config.ClientSecret.value,
9993
8990
  this.config.RedirectUri.value,
@@ -10014,12 +9011,12 @@ const RedditAds = (function() {
10014
9011
  * @param {string} refreshToken - The refresh token.
10015
9012
  * @returns {Object} An object with a success flag and either the access token or an error message.
10016
9013
  */
10017
- getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
9014
+ async getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
10018
9015
  const url = "https://www.reddit.com/api/v1/access_token";
10019
9016
  const headers = {
10020
9017
  "Content-Type": "application/x-www-form-urlencoded",
10021
9018
  "User-Agent": this.config.UserAgent.value,
10022
- "Authorization": "Basic " + EnvironmentAdapter3.base64Encode(clientId + ":" + clientSecret)
9019
+ "Authorization": "Basic " + CryptoUtils3.base64Encode(clientId + ":" + clientSecret)
10023
9020
  };
10024
9021
  const payload = {
10025
9022
  "grant_type": "refresh_token",
@@ -10035,8 +9032,8 @@ const RedditAds = (function() {
10035
9032
  muteHttpExceptions: true
10036
9033
  };
10037
9034
  try {
10038
- const response = EnvironmentAdapter3.fetch(url, options);
10039
- const result = response.getContentText();
9035
+ const response = await HttpUtils3.fetch(url, options);
9036
+ const result = await response.getContentText();
10040
9037
  const json = JSON.parse(result);
10041
9038
  if (json.error) {
10042
9039
  return { success: false, message: json.error };
@@ -10421,7 +9418,7 @@ const RedditAds = (function() {
10421
9418
  }
10422
9419
  };
10423
9420
  var RedditAdsConnector = class RedditAdsConnector extends AbstractConnector3 {
10424
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9421
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10425
9422
  super(config, source, null, runConfig);
10426
9423
  this.storageName = storageName;
10427
9424
  }
@@ -10429,12 +9426,12 @@ const RedditAds = (function() {
10429
9426
  * Main method - entry point for the import process
10430
9427
  * Processes all nodes defined in the fields configuration
10431
9428
  */
10432
- startImportProcess() {
9429
+ async startImportProcess() {
10433
9430
  const fields = RedditAdsHelper.parseFields(this.config.Fields.value);
10434
9431
  const accountIds = RedditAdsHelper.parseAccountIds(this.config.AccountIDs.value);
10435
9432
  for (const accountId of accountIds) {
10436
9433
  for (const nodeName in fields) {
10437
- this.processNode({
9434
+ await this.processNode({
10438
9435
  nodeName,
10439
9436
  accountId,
10440
9437
  fields: fields[nodeName] || []
@@ -10449,15 +9446,15 @@ const RedditAds = (function() {
10449
9446
  * @param {string} options.accountId - Account ID
10450
9447
  * @param {Array<string>} options.fields - Array of fields to fetch
10451
9448
  */
10452
- processNode({ nodeName, accountId, fields }) {
9449
+ async processNode({ nodeName, accountId, fields }) {
10453
9450
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
10454
- this.processTimeSeriesNode({
9451
+ await this.processTimeSeriesNode({
10455
9452
  nodeName,
10456
9453
  accountId,
10457
9454
  fields
10458
9455
  });
10459
9456
  } else {
10460
- this.processCatalogNode({
9457
+ await this.processCatalogNode({
10461
9458
  nodeName,
10462
9459
  accountId,
10463
9460
  fields
@@ -10472,7 +9469,7 @@ const RedditAds = (function() {
10472
9469
  * @param {Array<string>} options.fields - Array of fields to fetch
10473
9470
  * @param {Object} options.storage - Storage instance
10474
9471
  */
10475
- processTimeSeriesNode({ nodeName, accountId, fields }) {
9472
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
10476
9473
  var _a;
10477
9474
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
10478
9475
  if (daysToFetch <= 0) {
@@ -10482,13 +9479,14 @@ const RedditAds = (function() {
10482
9479
  for (let i = 0; i < daysToFetch; i++) {
10483
9480
  const currentDate = new Date(startDate);
10484
9481
  currentDate.setDate(currentDate.getDate() + i);
10485
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
9482
+ const formattedDate = DateUtils3.formatDate(currentDate);
10486
9483
  this.config.logMessage(`Start importing data for ${formattedDate}: ${accountId}/${nodeName}`);
10487
- const data = this.source.fetchData(nodeName, accountId, fields, currentDate);
9484
+ const data = await this.source.fetchData(nodeName, accountId, fields, currentDate);
10488
9485
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
10489
9486
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10490
9487
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10491
- this.getStorageByNode(nodeName).saveData(preparedData);
9488
+ const storage = await this.getStorageByNode(nodeName);
9489
+ await storage.saveData(preparedData);
10492
9490
  }
10493
9491
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
10494
9492
  this.config.updateLastRequstedDate(currentDate);
@@ -10503,13 +9501,14 @@ const RedditAds = (function() {
10503
9501
  * @param {Array<string>} options.fields - Array of fields to fetch
10504
9502
  * @param {Object} options.storage - Storage instance
10505
9503
  */
10506
- processCatalogNode({ nodeName, accountId, fields }) {
9504
+ async processCatalogNode({ nodeName, accountId, fields }) {
10507
9505
  var _a;
10508
- const data = this.source.fetchData(nodeName, accountId, fields);
9506
+ const data = await this.source.fetchData(nodeName, accountId, fields);
10509
9507
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for account ${accountId}` : `No records have been fetched`);
10510
9508
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10511
9509
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10512
- this.getStorageByNode(nodeName).saveData(preparedData);
9510
+ const storage = await this.getStorageByNode(nodeName);
9511
+ await storage.saveData(preparedData);
10513
9512
  }
10514
9513
  }
10515
9514
  /**
@@ -10517,7 +9516,7 @@ const RedditAds = (function() {
10517
9516
  * @param {string} nodeName - Name of the node
10518
9517
  * @returns {Object} Storage instance
10519
9518
  */
10520
- getStorageByNode(nodeName) {
9519
+ async getStorageByNode(nodeName) {
10521
9520
  if (!("storages" in this)) {
10522
9521
  this.storages = {};
10523
9522
  }
@@ -10535,6 +9534,7 @@ const RedditAds = (function() {
10535
9534
  this.source.fieldsSchema[nodeName].fields,
10536
9535
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
10537
9536
  );
9537
+ await this.storages[nodeName].init();
10538
9538
  }
10539
9539
  return this.storages[nodeName];
10540
9540
  }
@@ -10551,7 +9551,7 @@ const RedditAds = (function() {
10551
9551
  };
10552
9552
  })();
10553
9553
  const OpenHolidays = (function() {
10554
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
9554
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10555
9555
  var publicHolidaysFields = {
10556
9556
  id: {
10557
9557
  type: "string",
@@ -10659,10 +9659,10 @@ const OpenHolidays = (function() {
10659
9659
  * @param {string} [opts.end_time]
10660
9660
  * @returns {Array<Object>}
10661
9661
  */
10662
- fetchData({ nodeName, fields = [], start_time, end_time }) {
9662
+ async fetchData({ nodeName, fields = [], start_time, end_time }) {
10663
9663
  switch (nodeName) {
10664
9664
  case "publicHolidays":
10665
- return this._fetchPublicHolidays({ fields, start_time, end_time });
9665
+ return await this._fetchPublicHolidays({ fields, start_time, end_time });
10666
9666
  default:
10667
9667
  throw new Error(`Unknown node: ${nodeName}`);
10668
9668
  }
@@ -10675,10 +9675,10 @@ const OpenHolidays = (function() {
10675
9675
  * @param {string} options.end_time - End date for data fetch (YYYY-MM-DD format)
10676
9676
  * @returns {Array} Array of holiday data
10677
9677
  */
10678
- _fetchPublicHolidays({ fields, start_time, end_time }) {
9678
+ async _fetchPublicHolidays({ fields, start_time, end_time }) {
10679
9679
  let countryIsoCode = this.config.countryIsoCode.value;
10680
9680
  let languageIsoCode = this.config.languageIsoCode.value;
10681
- const holidays = this.makeRequest({
9681
+ const holidays = await this.makeRequest({
10682
9682
  endpoint: `PublicHolidays?countryIsoCode=${countryIsoCode}&languageIsoCode=${languageIsoCode}&validFrom=${start_time}&validTo=${end_time}`
10683
9683
  });
10684
9684
  if (!holidays || !holidays.length) {
@@ -10705,12 +9705,13 @@ const OpenHolidays = (function() {
10705
9705
  * @param {string} options.endpoint - API endpoint path (e.g., "PublicHolidays?countryIsoCode=CH&...")
10706
9706
  * @returns {Object} - API response parsed from JSON
10707
9707
  */
10708
- makeRequest({ endpoint }) {
9708
+ async makeRequest({ endpoint }) {
10709
9709
  const baseUrl = "https://openholidaysapi.org/";
10710
9710
  const url = `${baseUrl}${endpoint}`;
10711
9711
  console.log(`OpenHolidays API Request URL:`, url);
10712
- const response = EnvironmentAdapter3.fetch(url, { "method": "get", "muteHttpExceptions": true });
10713
- const result = JSON.parse(response.getContentText());
9712
+ const response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
9713
+ const text = await response.getContentText();
9714
+ const result = JSON.parse(text);
10714
9715
  return result;
10715
9716
  }
10716
9717
  /**
@@ -10736,7 +9737,7 @@ const OpenHolidays = (function() {
10736
9737
  }
10737
9738
  };
10738
9739
  var OpenHolidaysConnector = class OpenHolidaysConnector extends AbstractConnector3 {
10739
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9740
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10740
9741
  super(config, source, null, runConfig);
10741
9742
  this.storageName = storageName;
10742
9743
  }
@@ -10744,10 +9745,10 @@ const OpenHolidays = (function() {
10744
9745
  * Main method - entry point for the import process
10745
9746
  * Processes all nodes defined in the fields configuration
10746
9747
  */
10747
- startImportProcess() {
9748
+ async startImportProcess() {
10748
9749
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
10749
9750
  for (const nodeName in fields) {
10750
- this.processNode({
9751
+ await this.processNode({
10751
9752
  nodeName,
10752
9753
  fields: fields[nodeName] || []
10753
9754
  });
@@ -10759,12 +9760,12 @@ const OpenHolidays = (function() {
10759
9760
  * @param {string} options.nodeName - Name of the node to process
10760
9761
  * @param {Array<string>} options.fields - Array of fields to fetch
10761
9762
  */
10762
- processNode({ nodeName, fields }) {
9763
+ async processNode({ nodeName, fields }) {
10763
9764
  const storage = this.getStorageByNode(nodeName);
10764
9765
  if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
10765
- this.processTimeSeriesNode({ nodeName, fields, storage });
9766
+ await this.processTimeSeriesNode({ nodeName, fields, storage });
10766
9767
  } else {
10767
- this.processCatalogNode({ nodeName, fields, storage });
9768
+ await this.processCatalogNode({ nodeName, fields, storage });
10768
9769
  }
10769
9770
  }
10770
9771
  /**
@@ -10774,14 +9775,14 @@ const OpenHolidays = (function() {
10774
9775
  * @param {Array<string>} options.fields - Array of fields to fetch
10775
9776
  * @param {Object} options.storage - Storage instance
10776
9777
  */
10777
- processTimeSeriesNode({ nodeName, fields, storage }) {
9778
+ async processTimeSeriesNode({ nodeName, fields, storage }) {
10778
9779
  var _a;
10779
9780
  const dateRange = this.prepareDateRange();
10780
9781
  if (!dateRange) {
10781
9782
  console.log("No date range available for time series data");
10782
9783
  return;
10783
9784
  }
10784
- const data = this.source.fetchData({
9785
+ const data = await this.source.fetchData({
10785
9786
  nodeName,
10786
9787
  start_time: dateRange.startDate,
10787
9788
  end_time: dateRange.endDate,
@@ -10790,7 +9791,7 @@ const OpenHolidays = (function() {
10790
9791
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
10791
9792
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10792
9793
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10793
- storage.saveData(preparedData);
9794
+ await storage.saveData(preparedData);
10794
9795
  }
10795
9796
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
10796
9797
  this.config.updateLastRequstedDate(new Date(dateRange.endDate));
@@ -10803,7 +9804,7 @@ const OpenHolidays = (function() {
10803
9804
  * @param {Array<string>} options.fields - Array of fields to fetch
10804
9805
  * @param {Object} options.storage - Storage instance
10805
9806
  */
10806
- processCatalogNode({ nodeName, fields, storage }) {
9807
+ async processCatalogNode({ nodeName, fields, storage }) {
10807
9808
  console.log(`Catalog node processing not implemented for ${nodeName}`);
10808
9809
  }
10809
9810
  /**
@@ -10811,7 +9812,7 @@ const OpenHolidays = (function() {
10811
9812
  * @param {string} nodeName - Name of the node
10812
9813
  * @returns {Object} Storage instance
10813
9814
  */
10814
- getStorageByNode(nodeName) {
9815
+ async getStorageByNode(nodeName) {
10815
9816
  if (!("storages" in this)) {
10816
9817
  this.storages = {};
10817
9818
  }
@@ -10829,6 +9830,7 @@ const OpenHolidays = (function() {
10829
9830
  this.source.fieldsSchema[nodeName].fields,
10830
9831
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
10831
9832
  );
9833
+ await this.storages[nodeName].init();
10832
9834
  }
10833
9835
  return this.storages[nodeName];
10834
9836
  }
@@ -10844,8 +9846,8 @@ const OpenHolidays = (function() {
10844
9846
  const endDate = new Date(startDate);
10845
9847
  endDate.setDate(endDate.getDate() + daysToFetch - 1);
10846
9848
  return {
10847
- startDate: EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd"),
10848
- endDate: EnvironmentAdapter3.formatDate(endDate, "UTC", "yyyy-MM-dd")
9849
+ startDate: DateUtils3.formatDate(startDate),
9850
+ endDate: DateUtils3.formatDate(endDate)
10849
9851
  };
10850
9852
  }
10851
9853
  };
@@ -10861,7 +9863,7 @@ const OpenHolidays = (function() {
10861
9863
  };
10862
9864
  })();
10863
9865
  const OpenExchangeRates = (function() {
10864
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
9866
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10865
9867
  var historicalFields = {
10866
9868
  date: {
10867
9869
  type: "DATE",
@@ -10949,12 +9951,12 @@ const OpenExchangeRates = (function() {
10949
9951
  this.fieldsSchema = OpenExchangeRatesFieldsSchema;
10950
9952
  }
10951
9953
  /*
10952
- @param date The requested date as a Date object
9954
+ @param date The requested date as a Date object
10953
9955
 
10954
- @return data array
9956
+ @return data array
10955
9957
 
10956
- */
10957
- fetchData(date) {
9958
+ */
9959
+ async fetchData(date) {
10958
9960
  let data = [];
10959
9961
  let base = this.config.base.value;
10960
9962
  let app_id = this.config.AppId.value;
@@ -10962,13 +9964,14 @@ const OpenExchangeRates = (function() {
10962
9964
  if (this.config.Symbols.value) {
10963
9965
  symbols = "&symbols=" + String(this.config.Symbols.value).replace(/[^A-Z,]/g, "");
10964
9966
  }
10965
- var date = EnvironmentAdapter3.formatDate(date, "UTC", "yyyy-MM-dd");
9967
+ var date = DateUtils3.formatDate(date);
10966
9968
  const urlWithoutKey = `https://openexchangerates.org/api/historical/${date}.json?base=${base}${symbols}`;
10967
9969
  console.log(`OpenExchangeRates API URL:`, urlWithoutKey);
10968
9970
  const url = `${urlWithoutKey}&app_id=${app_id}`;
10969
9971
  this.config.logMessage(`Fetching rates for ${date}`);
10970
- var response = EnvironmentAdapter3.fetch(url, { "method": "get", "muteHttpExceptions": true });
10971
- var historical = JSON.parse(response.getContentText());
9972
+ var response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
9973
+ var text = await response.getContentText();
9974
+ var historical = JSON.parse(text);
10972
9975
  for (var currency in historical["rates"]) {
10973
9976
  data.push({
10974
9977
  date: new Date(date),
@@ -10981,7 +9984,7 @@ const OpenExchangeRates = (function() {
10981
9984
  }
10982
9985
  };
10983
9986
  var OpenExchangeRatesConnector = class OpenExchangeRatesConnector extends AbstractConnector3 {
10984
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9987
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10985
9988
  super(config, source, null, runConfig);
10986
9989
  this.storageName = storageName;
10987
9990
  }
@@ -10994,11 +9997,11 @@ const OpenExchangeRates = (function() {
10994
9997
  * Main method - entry point for the import process
10995
9998
  * Processes all nodes defined in the fields configuration
10996
9999
  */
10997
- startImportProcess() {
10000
+ async startImportProcess() {
10998
10001
  var _a;
10999
10002
  const fields = ConnectorUtils.parseFields((_a = this.config.Fields) == null ? void 0 : _a.value);
11000
10003
  for (const nodeName in fields) {
11001
- this.processNode({
10004
+ await this.processNode({
11002
10005
  nodeName,
11003
10006
  fields: fields[nodeName] || []
11004
10007
  });
@@ -11010,11 +10013,11 @@ const OpenExchangeRates = (function() {
11010
10013
  * @param {string} options.nodeName - Name of the node to process
11011
10014
  * @param {Array<string>} options.fields - Array of fields to fetch
11012
10015
  */
11013
- processNode({ nodeName, fields }) {
10016
+ async processNode({ nodeName, fields }) {
11014
10017
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
11015
- this.processTimeSeriesNode({ nodeName, fields });
10018
+ await this.processTimeSeriesNode({ nodeName, fields });
11016
10019
  } else {
11017
- this.processCatalogNode({ nodeName, fields });
10020
+ await this.processCatalogNode({ nodeName, fields });
11018
10021
  }
11019
10022
  }
11020
10023
  /**
@@ -11023,7 +10026,7 @@ const OpenExchangeRates = (function() {
11023
10026
  * @param {string} options.nodeName - Name of the node
11024
10027
  * @param {Array<string>} options.fields - Array of fields to fetch
11025
10028
  */
11026
- processTimeSeriesNode({ nodeName, fields }) {
10029
+ async processTimeSeriesNode({ nodeName, fields }) {
11027
10030
  var _a;
11028
10031
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
11029
10032
  if (daysToFetch <= 0) {
@@ -11033,11 +10036,12 @@ const OpenExchangeRates = (function() {
11033
10036
  for (let daysShift = 0; daysShift < daysToFetch; daysShift++) {
11034
10037
  const currentDate = new Date(startDate);
11035
10038
  currentDate.setDate(currentDate.getDate() + daysShift);
11036
- let data = this.source.fetchData(currentDate);
10039
+ let data = await this.source.fetchData(currentDate);
11037
10040
  this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
11038
10041
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
11039
10042
  const preparedData = data.length ? data : [];
11040
- this.getStorageByNode(nodeName).saveData(preparedData);
10043
+ const storage = await this.getStorageByNode(nodeName);
10044
+ await storage.saveData(preparedData);
11041
10045
  }
11042
10046
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
11043
10047
  this.config.updateLastRequstedDate(currentDate);
@@ -11050,7 +10054,7 @@ const OpenExchangeRates = (function() {
11050
10054
  * @param {string} options.nodeName - Name of the node
11051
10055
  * @param {Array<string>} options.fields - Array of fields to fetch
11052
10056
  */
11053
- processCatalogNode({ nodeName, fields }) {
10057
+ async processCatalogNode({ nodeName, fields }) {
11054
10058
  console.log(`Catalog node processing not implemented for ${nodeName}`);
11055
10059
  }
11056
10060
  //---- getStorageName -------------------------------------------------
@@ -11062,7 +10066,7 @@ const OpenExchangeRates = (function() {
11062
10066
  * @return AbstractStorage
11063
10067
  *
11064
10068
  */
11065
- getStorageByNode(nodeName) {
10069
+ async getStorageByNode(nodeName) {
11066
10070
  if (!("storages" in this)) {
11067
10071
  this.storages = {};
11068
10072
  }
@@ -11080,6 +10084,7 @@ const OpenExchangeRates = (function() {
11080
10084
  this.source.fieldsSchema[nodeName]["fields"],
11081
10085
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
11082
10086
  );
10087
+ await this.storages[nodeName].init();
11083
10088
  }
11084
10089
  return this.storages[nodeName];
11085
10090
  }
@@ -11096,7 +10101,7 @@ const OpenExchangeRates = (function() {
11096
10101
  };
11097
10102
  })();
11098
10103
  const MicrosoftAds = (function() {
11099
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10104
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
11100
10105
  const MicrosoftAdsHelper = {
11101
10106
  /**
11102
10107
  * Parse fields string into a structured object
@@ -11119,7 +10124,7 @@ const MicrosoftAds = (function() {
11119
10124
  * @param {number} [opts.interval=5000]
11120
10125
  * @returns {Object}
11121
10126
  */
11122
- pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
10127
+ async pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
11123
10128
  const startTime = Date.now();
11124
10129
  const timeout = 15 * 60 * 1e3;
11125
10130
  let statusResult;
@@ -11128,9 +10133,10 @@ const MicrosoftAds = (function() {
11128
10133
  if (Date.now() - startTime > timeout) {
11129
10134
  throw new Error("Polling timed out after 15 minutes");
11130
10135
  }
11131
- EnvironmentAdapter3.sleep(interval);
11132
- const response = EnvironmentAdapter3.fetch(url, options);
11133
- statusResult = JSON.parse(response.getContentText());
10136
+ await AsyncUtils3.delay(interval);
10137
+ const response = await HttpUtils3.fetch(url, options);
10138
+ const text = await response.getContentText();
10139
+ statusResult = JSON.parse(text);
11134
10140
  } while (!isDone(statusResult));
11135
10141
  return statusResult;
11136
10142
  } catch (error) {
@@ -11146,13 +10152,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
11146
10152
  * @param {string} url
11147
10153
  * @returns {Array<Array<string>>}
11148
10154
  */
11149
- downloadCsvRows(url) {
11150
- const response = EnvironmentAdapter3.fetch(url);
11151
- const files = EnvironmentAdapter3.unzip(response.getBlob());
10155
+ async downloadCsvRows(url) {
10156
+ const response = await HttpUtils3.fetch(url);
10157
+ const blob = await response.getBlob();
10158
+ const files = FileUtils3.unzip(blob);
11152
10159
  const allRows = [];
11153
10160
  files.forEach((file) => {
11154
10161
  const csvText = file.getDataAsString();
11155
- const rows = EnvironmentAdapter3.parseCsv(csvText);
10162
+ const rows = FileUtils3.parseCsv(csvText);
11156
10163
  allRows.push(...rows);
11157
10164
  });
11158
10165
  return allRows;
@@ -12279,7 +11286,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12279
11286
  /**
12280
11287
  * Retrieve and store an OAuth access token using the refresh token
12281
11288
  */
12282
- getAccessToken() {
11289
+ async getAccessToken() {
12283
11290
  var _a, _b;
12284
11291
  const tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
12285
11292
  const scopes = [
@@ -12305,8 +11312,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12305
11312
  body: Object.entries(form).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
12306
11313
  // TODO: body is for Node.js; refactor to centralize JSON option creation
12307
11314
  };
12308
- const resp = EnvironmentAdapter3.fetch(tokenUrl, options);
12309
- const json = JSON.parse(resp.getContentText());
11315
+ const resp = await HttpUtils3.fetch(tokenUrl, options);
11316
+ const text = await resp.getContentText();
11317
+ const json = JSON.parse(text);
12310
11318
  if (json.error) {
12311
11319
  throw new Error(`Token error: ${json.error} - ${json.error_description}`);
12312
11320
  }
@@ -12332,7 +11340,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12332
11340
  * @param {Function} [opts.onBatchReady] - Optional callback for batch processing
12333
11341
  * @returns {Array<Object>}
12334
11342
  */
12335
- fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
11343
+ async fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
12336
11344
  const schema = this.fieldsSchema[nodeName];
12337
11345
  if (schema.uniqueKeys) {
12338
11346
  const missingKeys = schema.uniqueKeys.filter((key) => !fields.includes(key));
@@ -12342,12 +11350,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12342
11350
  }
12343
11351
  switch (nodeName) {
12344
11352
  case "campaigns":
12345
- this._fetchCampaignData({ accountId, fields, onBatchReady });
11353
+ await this._fetchCampaignData({ accountId, fields, onBatchReady });
12346
11354
  return [];
12347
11355
  case "ad_performance_report":
12348
- return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
11356
+ return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
12349
11357
  case "user_location_performance_report":
12350
- return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
11358
+ return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
12351
11359
  default:
12352
11360
  throw new Error(`Unknown node: ${nodeName}`);
12353
11361
  }
@@ -12361,14 +11369,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12361
11369
  * @returns {void}
12362
11370
  * @private
12363
11371
  */
12364
- _fetchCampaignData({ accountId, fields, onBatchReady }) {
12365
- this.getAccessToken();
11372
+ async _fetchCampaignData({ accountId, fields, onBatchReady }) {
11373
+ await this.getAccessToken();
12366
11374
  this.config.logMessage(`Fetching Campaigns, AssetGroups and AdGroups for account ${accountId}...`);
12367
11375
  const entityTypes = ["Campaigns", "AssetGroups", "AdGroups"];
12368
11376
  const allRecords = [];
12369
11377
  let campaignRecords = [];
12370
11378
  for (const entityType of entityTypes) {
12371
- const records = this._downloadEntity({
11379
+ const records = await this._downloadEntity({
12372
11380
  submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByAccountIds",
12373
11381
  submitOpts: {
12374
11382
  method: "post",
@@ -12406,21 +11414,21 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12406
11414
  }
12407
11415
  const filteredMainData = MicrosoftAdsHelper.filterByFields(allRecords, fields);
12408
11416
  if (filteredMainData.length > 0) {
12409
- onBatchReady(filteredMainData);
11417
+ await onBatchReady(filteredMainData);
12410
11418
  }
12411
11419
  this.config.logMessage(`Fetching Keywords for account ${accountId} (processing by campaigns to avoid size limits)...`);
12412
11420
  const campaignIds = MicrosoftAdsHelper.extractCampaignIds(campaignRecords);
12413
11421
  this.config.logMessage(`Found ${campaignIds.length} campaigns, fetching Keywords in batches`);
12414
11422
  this.config.logMessage(`Campaign IDs: ${campaignIds.slice(0, 10).join(", ")}${campaignIds.length > 10 ? "..." : ""}`);
12415
11423
  let totalFetched = 0;
12416
- this._fetchEntityByCampaigns({
11424
+ await this._fetchEntityByCampaigns({
12417
11425
  accountId,
12418
11426
  entityType: "Keywords",
12419
11427
  campaignIds,
12420
- onBatchReady: (batchRecords) => {
11428
+ onBatchReady: async (batchRecords) => {
12421
11429
  totalFetched += batchRecords.length;
12422
11430
  const filteredBatch = MicrosoftAdsHelper.filterByFields(batchRecords, fields);
12423
- onBatchReady(filteredBatch);
11431
+ await onBatchReady(filteredBatch);
12424
11432
  }
12425
11433
  });
12426
11434
  this.config.logMessage(`${totalFetched} rows of Keywords were fetched for account ${accountId}`);
@@ -12434,15 +11442,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12434
11442
  * @returns {Array<Object>}
12435
11443
  * @private
12436
11444
  */
12437
- _downloadEntity({ submitUrl, submitOpts }) {
12438
- const submitResp = EnvironmentAdapter3.fetch(submitUrl, submitOpts);
12439
- const requestId = JSON.parse(submitResp.getContentText()).DownloadRequestId;
11445
+ async _downloadEntity({ submitUrl, submitOpts }) {
11446
+ const submitResp = await HttpUtils3.fetch(submitUrl, submitOpts);
11447
+ const text = await submitResp.getContentText();
11448
+ const requestId = JSON.parse(text).DownloadRequestId;
12440
11449
  const pollUrl = "https://bulk.api.bingads.microsoft.com/Bulk/v13/BulkDownloadStatus/Query";
12441
11450
  const pollOpts = Object.assign({}, submitOpts, {
12442
11451
  payload: JSON.stringify({ RequestId: requestId }),
12443
11452
  body: JSON.stringify({ RequestId: requestId })
12444
11453
  });
12445
- const pollResult = MicrosoftAdsHelper.pollUntilStatus({
11454
+ const pollResult = await MicrosoftAdsHelper.pollUntilStatus({
12446
11455
  url: pollUrl,
12447
11456
  options: pollOpts,
12448
11457
  isDone: (status) => {
@@ -12452,7 +11461,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12452
11461
  return status.RequestStatus === "Completed";
12453
11462
  }
12454
11463
  });
12455
- const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
11464
+ const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
12456
11465
  const result = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
12457
11466
  return result;
12458
11467
  }
@@ -12466,7 +11475,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12466
11475
  * @returns {Array<Object>} - Returns empty array if onBatchReady callback is provided, otherwise returns all records
12467
11476
  * @private
12468
11477
  */
12469
- _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
11478
+ async _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
12470
11479
  if (campaignIds.length === 0) {
12471
11480
  this.config.logMessage(`No active campaigns found for account ${accountId}, skipping ${entityType} fetch`);
12472
11481
  return [];
@@ -12476,9 +11485,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12476
11485
  const campaignBatch = campaignIds.slice(i, i + batchSize);
12477
11486
  this.config.logMessage(`Fetching ${entityType} for campaigns batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(campaignIds.length / batchSize)} (${campaignBatch.length} campaigns)`);
12478
11487
  try {
12479
- const batchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch });
11488
+ const batchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch });
12480
11489
  this.config.logMessage(`Fetched ${batchRecords.length} ${entityType.toLowerCase()} from current batch`);
12481
- onBatchReady(batchRecords);
11490
+ await onBatchReady(batchRecords);
12482
11491
  } catch (error) {
12483
11492
  if (error.message && error.message.includes("100MB")) {
12484
11493
  const newBatchSize = Math.max(1, Math.floor(batchSize / 2));
@@ -12486,9 +11495,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12486
11495
  for (let j = i; j < Math.min(i + batchSize, campaignIds.length); j += newBatchSize) {
12487
11496
  const smallerBatch = campaignIds.slice(j, j + newBatchSize);
12488
11497
  try {
12489
- const smallerBatchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
11498
+ const smallerBatchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
12490
11499
  this.config.logMessage(`Fetched ${smallerBatchRecords.length} ${entityType.toLowerCase()} from smaller batch (${smallerBatch.length} campaigns)`);
12491
- onBatchReady(smallerBatchRecords);
11500
+ await onBatchReady(smallerBatchRecords);
12492
11501
  } catch (smallerError) {
12493
11502
  if (smallerError.message && smallerError.message.includes("100MB")) {
12494
11503
  throw new Error(`Failed to fetch ${entityType}: batch size of ${smallerBatch.length} campaigns still exceeds 100MB limit`);
@@ -12515,7 +11524,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12515
11524
  * @returns {Array<Object>}
12516
11525
  * @private
12517
11526
  */
12518
- _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
11527
+ async _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
12519
11528
  const downloadBody = {
12520
11529
  Campaigns: campaignBatch.map((id) => ({
12521
11530
  CampaignId: Number(id),
@@ -12540,7 +11549,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12540
11549
  payload: JSON.stringify(downloadBody),
12541
11550
  body: JSON.stringify(downloadBody)
12542
11551
  };
12543
- return this._downloadEntity({
11552
+ return await this._downloadEntity({
12544
11553
  submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByCampaignIds",
12545
11554
  submitOpts
12546
11555
  });
@@ -12556,22 +11565,22 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12556
11565
  * @returns {Array<Object>}
12557
11566
  * @private
12558
11567
  */
12559
- _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
12560
- this.getAccessToken();
11568
+ async _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
11569
+ await this.getAccessToken();
12561
11570
  const schema = this.fieldsSchema[nodeName];
12562
- const submitResponse = this._submitReportRequest({
11571
+ const submitResponse = await this._submitReportRequest({
12563
11572
  accountId,
12564
11573
  fields,
12565
11574
  start_time,
12566
11575
  end_time,
12567
11576
  schema
12568
11577
  });
12569
- const pollResult = this._pollReportStatus({ submitResponse });
11578
+ const pollResult = await this._pollReportStatus({ submitResponse });
12570
11579
  if (!pollResult.ReportRequestStatus.ReportDownloadUrl) {
12571
11580
  this.config.logMessage(`No data available for the specified time period (${start_time} to ${end_time}). Report status: ${JSON.stringify(pollResult.ReportRequestStatus)}`);
12572
11581
  return [];
12573
11582
  }
12574
- const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
11583
+ const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
12575
11584
  const records = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
12576
11585
  return MicrosoftAdsHelper.filterByFields(records, fields);
12577
11586
  }
@@ -12586,7 +11595,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12586
11595
  * @returns {Object} - Submit response
12587
11596
  * @private
12588
11597
  */
12589
- _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
11598
+ async _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
12590
11599
  const dateRange = {
12591
11600
  CustomDateRangeStart: { Day: new Date(start_time).getDate(), Month: new Date(start_time).getMonth() + 1, Year: new Date(start_time).getFullYear() },
12592
11601
  CustomDateRangeEnd: { Day: new Date(end_time).getDate(), Month: new Date(end_time).getMonth() + 1, Year: new Date(end_time).getFullYear() },
@@ -12619,8 +11628,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12619
11628
  body: JSON.stringify({ ReportRequest: requestBody })
12620
11629
  // TODO: body is for Node.js; refactor to centralize JSON option creation
12621
11630
  };
12622
- const submitResp = EnvironmentAdapter3.fetch(submitUrl, submitOpts);
12623
- const submitResponseText = submitResp.getContentText();
11631
+ const submitResp = await HttpUtils3.fetch(submitUrl, submitOpts);
11632
+ const submitResponseText = await submitResp.getContentText();
12624
11633
  try {
12625
11634
  const submitResponse = JSON.parse(submitResponseText);
12626
11635
  if (submitResponse.OperationErrors && submitResponse.OperationErrors.length > 0) {
@@ -12644,7 +11653,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12644
11653
  * @returns {Object} - Poll result with report status
12645
11654
  * @private
12646
11655
  */
12647
- _pollReportStatus({ submitResponse }) {
11656
+ async _pollReportStatus({ submitResponse }) {
12648
11657
  const pollUrl = "https://reporting.api.bingads.microsoft.com/Reporting/v13/GenerateReport/Poll";
12649
11658
  const submitResponseText = JSON.stringify(submitResponse);
12650
11659
  const pollOpts = {
@@ -12660,7 +11669,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12660
11669
  payload: submitResponseText,
12661
11670
  body: submitResponseText
12662
11671
  };
12663
- return MicrosoftAdsHelper.pollUntilStatus({
11672
+ return await MicrosoftAdsHelper.pollUntilStatus({
12664
11673
  url: pollUrl,
12665
11674
  options: pollOpts,
12666
11675
  isDone: (status) => {
@@ -12673,7 +11682,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12673
11682
  }
12674
11683
  };
12675
11684
  var MicrosoftAdsConnector = class MicrosoftAdsConnector extends AbstractConnector3 {
12676
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
11685
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
12677
11686
  super(config, source, null, runConfig);
12678
11687
  this.storageName = storageName;
12679
11688
  }
@@ -12681,10 +11690,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12681
11690
  * Main method - entry point for the import process
12682
11691
  * Processes all nodes defined in the fields configuration
12683
11692
  */
12684
- startImportProcess() {
11693
+ async startImportProcess() {
12685
11694
  const fields = MicrosoftAdsHelper.parseFields(this.config.Fields.value);
12686
11695
  for (const nodeName in fields) {
12687
- this.processNode({
11696
+ await this.processNode({
12688
11697
  nodeName,
12689
11698
  accountId: this.config.AccountID.value,
12690
11699
  fields: fields[nodeName] || []
@@ -12698,15 +11707,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12698
11707
  * @param {string} options.accountId - Account ID
12699
11708
  * @param {Array<string>} options.fields - Array of fields to fetch
12700
11709
  */
12701
- processNode({ nodeName, accountId, fields }) {
11710
+ async processNode({ nodeName, accountId, fields }) {
12702
11711
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
12703
- this.processTimeSeriesNode({
11712
+ await this.processTimeSeriesNode({
12704
11713
  nodeName,
12705
11714
  accountId,
12706
11715
  fields
12707
11716
  });
12708
11717
  } else {
12709
- this.processCatalogNode({
11718
+ await this.processCatalogNode({
12710
11719
  nodeName,
12711
11720
  accountId,
12712
11721
  fields
@@ -12721,7 +11730,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12721
11730
  * @param {Array<string>} options.fields - Array of fields to fetch
12722
11731
  * @param {Object} options.storage - Storage instance
12723
11732
  */
12724
- processTimeSeriesNode({ nodeName, accountId, fields }) {
11733
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
12725
11734
  var _a;
12726
11735
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
12727
11736
  if (daysToFetch <= 0) {
@@ -12731,9 +11740,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12731
11740
  for (let dayOffset = 0; dayOffset < daysToFetch; dayOffset++) {
12732
11741
  const currentDate = new Date(startDate);
12733
11742
  currentDate.setDate(currentDate.getDate() + dayOffset);
12734
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
11743
+ const formattedDate = DateUtils3.formatDate(currentDate);
12735
11744
  this.config.logMessage(`Processing ${nodeName} for ${accountId} on ${formattedDate} (day ${dayOffset + 1} of ${daysToFetch})`);
12736
- const data = this.source.fetchData({
11745
+ const data = await this.source.fetchData({
12737
11746
  nodeName,
12738
11747
  accountId,
12739
11748
  start_time: formattedDate,
@@ -12743,7 +11752,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12743
11752
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
12744
11753
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
12745
11754
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
12746
- this.getStorageByNode(nodeName).saveData(preparedData);
11755
+ const storage = await this.getStorageByNode(nodeName);
11756
+ await storage.saveData(preparedData);
12747
11757
  data.length && this.config.logMessage(`Successfully saved ${data.length} rows for ${formattedDate}`);
12748
11758
  }
12749
11759
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
@@ -12759,22 +11769,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12759
11769
  * @param {Array<string>} options.fields - Array of fields to fetch
12760
11770
  * @param {Object} options.storage - Storage instance
12761
11771
  */
12762
- processCatalogNode({ nodeName, accountId, fields }) {
11772
+ async processCatalogNode({ nodeName, accountId, fields }) {
12763
11773
  var _a;
12764
- const data = this.source.fetchData({
11774
+ const data = await this.source.fetchData({
12765
11775
  nodeName,
12766
11776
  accountId,
12767
11777
  fields,
12768
- onBatchReady: (batchData) => {
11778
+ onBatchReady: async (batchData) => {
12769
11779
  this.config.logMessage(`Saving batch of ${batchData.length} records to storage`);
12770
11780
  const preparedData = this.addMissingFieldsToData(batchData, fields);
12771
- this.getStorageByNode(nodeName).saveData(preparedData);
11781
+ const storage = await this.getStorageByNode(nodeName);
11782
+ await storage.saveData(preparedData);
12772
11783
  }
12773
11784
  });
12774
11785
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
12775
11786
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
12776
11787
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
12777
- this.getStorageByNode(nodeName).saveData(preparedData);
11788
+ const storage = await this.getStorageByNode(nodeName);
11789
+ await storage.saveData(preparedData);
12778
11790
  }
12779
11791
  }
12780
11792
  /**
@@ -12782,7 +11794,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12782
11794
  * @param {string} nodeName - Name of the node
12783
11795
  * @returns {Object} Storage instance
12784
11796
  */
12785
- getStorageByNode(nodeName) {
11797
+ async getStorageByNode(nodeName) {
12786
11798
  if (!("storages" in this)) {
12787
11799
  this.storages = {};
12788
11800
  }
@@ -12800,6 +11812,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12800
11812
  this.source.fieldsSchema[nodeName].fields,
12801
11813
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
12802
11814
  );
11815
+ await this.storages[nodeName].init();
12803
11816
  }
12804
11817
  return this.storages[nodeName];
12805
11818
  }
@@ -12816,7 +11829,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12816
11829
  };
12817
11830
  })();
12818
11831
  const LinkedInPages = (function() {
12819
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
11832
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
12820
11833
  var followerStatisticsTimeBoundFields = {
12821
11834
  "organization_urn": {
12822
11835
  "description": "Organization URN",
@@ -12998,7 +12011,7 @@ const LinkedInPages = (function() {
12998
12011
  * @param {Object} params - Additional parameters for the request
12999
12012
  * @returns {Array} - Array of processed data objects
13000
12013
  */
13001
- fetchData(nodeName, urn, params = {}) {
12014
+ async fetchData(nodeName, urn, params = {}) {
13002
12015
  var _a;
13003
12016
  const fields = params.fields || [];
13004
12017
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -13008,7 +12021,7 @@ const LinkedInPages = (function() {
13008
12021
  }
13009
12022
  switch (nodeName) {
13010
12023
  case "follower_statistics_time_bound":
13011
- return this.fetchOrganizationStats({
12024
+ return await this.fetchOrganizationStats({
13012
12025
  urn,
13013
12026
  nodeName,
13014
12027
  endpoint: "organizationalEntityFollowerStatistics",
@@ -13017,7 +12030,7 @@ const LinkedInPages = (function() {
13017
12030
  params
13018
12031
  });
13019
12032
  case "follower_statistics":
13020
- return this.fetchOrganizationStats({
12033
+ return await this.fetchOrganizationStats({
13021
12034
  urn,
13022
12035
  nodeName,
13023
12036
  endpoint: "organizationalEntityFollowerStatistics",
@@ -13042,7 +12055,7 @@ const LinkedInPages = (function() {
13042
12055
  * @param {Array} [options.params.fields] - Additional parameters including fields
13043
12056
  * @returns {Array} - Processed statistics data
13044
12057
  */
13045
- fetchOrganizationStats(options) {
12058
+ async fetchOrganizationStats(options) {
13046
12059
  const { urn, nodeName, endpoint, entityParam, formatter, params } = options;
13047
12060
  const orgUrn = `urn:li:organization:${urn}`;
13048
12061
  const encodedUrn = encodeURIComponent(orgUrn);
@@ -13053,7 +12066,7 @@ const LinkedInPages = (function() {
13053
12066
  const endTimestamp = new Date(params.endDate).getTime();
13054
12067
  url += `&timeIntervals=(timeRange:(start:${startTimestamp},end:${endTimestamp}),timeGranularityType:DAY)`;
13055
12068
  }
13056
- const response = this.makeRequest(url);
12069
+ const response = await this.makeRequest(url);
13057
12070
  const elements = response.elements || [];
13058
12071
  if (elements.length === 0) {
13059
12072
  return [];
@@ -13070,9 +12083,9 @@ const LinkedInPages = (function() {
13070
12083
  * @param {Object} headers - Optional additional headers
13071
12084
  * @returns {Object} - API response parsed from JSON
13072
12085
  */
13073
- makeRequest(url) {
12086
+ async makeRequest(url) {
13074
12087
  console.log(`LinkedIn Pages API URL:`, url);
13075
- OAuthUtils.getAccessToken({
12088
+ await OAuthUtils.getAccessToken({
13076
12089
  config: this.config,
13077
12090
  tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
13078
12091
  formData: {
@@ -13087,12 +12100,13 @@ const LinkedInPages = (function() {
13087
12100
  "X-RestLi-Protocol-Version": "2.0.0"
13088
12101
  };
13089
12102
  const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
13090
- const response = EnvironmentAdapter3.fetch(authUrl, { headers });
13091
- const result = JSON.parse(response.getContentText());
13092
- if (result.status && result.status >= HTTP_STATUS2.BAD_REQUEST) {
13093
- throw new Error(`LinkedIn API Error: ${result.message || "Unknown error"} (Status: ${result.status})`);
12103
+ const response = await HttpUtils3.fetch(authUrl, { headers });
12104
+ const result = await response.getContentText();
12105
+ const parsedResult = JSON.parse(result);
12106
+ if (parsedResult.status && parsedResult.status >= HTTP_STATUS2.BAD_REQUEST) {
12107
+ throw new Error(`LinkedIn API Error: ${parsedResult.message || "Unknown error"} (Status: ${parsedResult.status})`);
13094
12108
  }
13095
- return result;
12109
+ return parsedResult;
13096
12110
  }
13097
12111
  /**
13098
12112
  * Process time-bound statistics data
@@ -13186,7 +12200,7 @@ const LinkedInPages = (function() {
13186
12200
  }
13187
12201
  };
13188
12202
  var LinkedInPagesConnector = class LinkedInPagesConnector extends AbstractConnector3 {
13189
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
12203
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
13190
12204
  super(config, source, null, runConfig);
13191
12205
  this.storageName = storageName;
13192
12206
  }
@@ -13194,11 +12208,11 @@ const LinkedInPages = (function() {
13194
12208
  * Main method - entry point for the import process
13195
12209
  * Processes all nodes defined in the fields configuration
13196
12210
  */
13197
- startImportProcess() {
12211
+ async startImportProcess() {
13198
12212
  const urns = FormatUtils.parseIds(this.config.OrganizationURNs.value, { prefix: "urn:li:organization:" });
13199
12213
  const dataSources = FormatUtils.parseFields(this.config.Fields.value);
13200
12214
  for (const nodeName in dataSources) {
13201
- this.processNode({
12215
+ await this.processNode({
13202
12216
  nodeName,
13203
12217
  urns,
13204
12218
  fields: dataSources[nodeName] || []
@@ -13212,13 +12226,13 @@ const LinkedInPages = (function() {
13212
12226
  * @param {Array} options.urns - URNs to process
13213
12227
  * @param {Array} options.fields - Fields to fetch
13214
12228
  */
13215
- processNode({ nodeName, urns, fields }) {
12229
+ async processNode({ nodeName, urns, fields }) {
13216
12230
  const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
13217
12231
  const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
13218
12232
  if (isTimeSeriesNode && !dateInfo) {
13219
12233
  return;
13220
12234
  }
13221
- this.fetchAndSaveData({
12235
+ await this.fetchAndSaveData({
13222
12236
  nodeName,
13223
12237
  urns,
13224
12238
  fields,
@@ -13239,7 +12253,7 @@ const LinkedInPages = (function() {
13239
12253
  * @param {string} [options.startDate] - Start date for time series data
13240
12254
  * @param {string} [options.endDate] - End date for time series data
13241
12255
  */
13242
- fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
12256
+ async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
13243
12257
  var _a;
13244
12258
  for (const urn of urns) {
13245
12259
  console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
@@ -13247,11 +12261,12 @@ const LinkedInPages = (function() {
13247
12261
  console.log(`End date is +1 day due to LinkedIn Pages API requirements (to include actual end date in results)`);
13248
12262
  }
13249
12263
  const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
13250
- const data = this.source.fetchData(nodeName, urn, params);
12264
+ const data = await this.source.fetchData(nodeName, urn, params);
13251
12265
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
13252
12266
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
13253
12267
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
13254
- this.getStorageByNode(nodeName).saveData(preparedData);
12268
+ const storage = await this.getStorageByNode(nodeName);
12269
+ await storage.saveData(preparedData);
13255
12270
  }
13256
12271
  }
13257
12272
  }
@@ -13260,7 +12275,7 @@ const LinkedInPages = (function() {
13260
12275
  * @param {string} nodeName - Name of the node
13261
12276
  * @returns {Object} - Storage instance
13262
12277
  */
13263
- getStorageByNode(nodeName) {
12278
+ async getStorageByNode(nodeName) {
13264
12279
  if (!("storages" in this)) {
13265
12280
  this.storages = {};
13266
12281
  }
@@ -13278,6 +12293,7 @@ const LinkedInPages = (function() {
13278
12293
  this.source.fieldsSchema[nodeName]["fields"],
13279
12294
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
13280
12295
  );
12296
+ await this.storages[nodeName].init();
13281
12297
  }
13282
12298
  return this.storages[nodeName];
13283
12299
  }
@@ -13315,7 +12331,7 @@ const LinkedInPages = (function() {
13315
12331
  };
13316
12332
  })();
13317
12333
  const LinkedInAds = (function() {
13318
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
12334
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
13319
12335
  var creativesFields = {
13320
12336
  "account": {
13321
12337
  "description": "URN identifying the advertising account associated with the creative. This field is read-only.",
@@ -14271,7 +13287,7 @@ const LinkedInAds = (function() {
14271
13287
  * @param {Object} params - Additional parameters for the request
14272
13288
  * @returns {Array} - Array of fetched data objects
14273
13289
  */
14274
- fetchData(nodeName, urn, params = {}) {
13290
+ async fetchData(nodeName, urn, params = {}) {
14275
13291
  var _a;
14276
13292
  const fields = params.fields || [];
14277
13293
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -14281,15 +13297,15 @@ const LinkedInAds = (function() {
14281
13297
  }
14282
13298
  switch (nodeName) {
14283
13299
  case "adAccounts":
14284
- return this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
13300
+ return await this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
14285
13301
  case "adCampaignGroups":
14286
- return this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
13302
+ return await this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
14287
13303
  case "adCampaigns":
14288
- return this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
13304
+ return await this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
14289
13305
  case "creatives":
14290
- return this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
13306
+ return await this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
14291
13307
  case "adAnalytics":
14292
- return this.fetchAdAnalytics(urn, params);
13308
+ return await this.fetchAdAnalytics(urn, params);
14293
13309
  default:
14294
13310
  throw new Error(`Unknown node: ${nodeName}`);
14295
13311
  }
@@ -14302,10 +13318,10 @@ const LinkedInAds = (function() {
14302
13318
  * @param {Object} options.params - Additional parameters for the request
14303
13319
  * @returns {Array} - Array containing the single resource
14304
13320
  */
14305
- fetchSingleResource({ urn, resourceType, params }) {
13321
+ async fetchSingleResource({ urn, resourceType, params }) {
14306
13322
  let url = `${this.BASE_URL}${resourceType}/${encodeURIComponent(urn)}`;
14307
13323
  url += `?fields=${this.formatFields(params.fields)}`;
14308
- const result = this.makeRequest(url);
13324
+ const result = await this.makeRequest(url);
14309
13325
  return [result];
14310
13326
  }
14311
13327
  /**
@@ -14317,10 +13333,10 @@ const LinkedInAds = (function() {
14317
13333
  * @param {string} [options.queryType='search'] - Query type parameter
14318
13334
  * @returns {Array} - Array of fetched resources
14319
13335
  */
14320
- fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
13336
+ async fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
14321
13337
  let url = `${this.BASE_URL}adAccounts/${encodeURIComponent(urn)}/${resourceType}?q=${queryType}&pageSize=100`;
14322
13338
  url += `&fields=${this.formatFields(params.fields)}`;
14323
- return this.fetchWithPagination(url);
13339
+ return await this.fetchWithPagination(url);
14324
13340
  }
14325
13341
  /**
14326
13342
  * Fetch analytics data, handling field limits and data merging
@@ -14331,7 +13347,7 @@ const LinkedInAds = (function() {
14331
13347
  * @param {Array} params.fields - Fields to fetch
14332
13348
  * @returns {Array} - Combined array of analytics data
14333
13349
  */
14334
- fetchAdAnalytics(urn, params) {
13350
+ async fetchAdAnalytics(urn, params) {
14335
13351
  const startDate = new Date(params.startDate);
14336
13352
  const endDate = new Date(params.endDate);
14337
13353
  const accountUrn = `urn:li:sponsoredAccount:${urn}`;
@@ -14346,7 +13362,7 @@ const LinkedInAds = (function() {
14346
13362
  encodedUrn,
14347
13363
  fields: fieldChunk
14348
13364
  });
14349
- const res = this.makeRequest(url);
13365
+ const res = await this.makeRequest(url);
14350
13366
  const elements = res.elements || [];
14351
13367
  allResults = this.mergeAnalyticsResults(allResults, elements);
14352
13368
  }
@@ -14475,9 +13491,9 @@ const LinkedInAds = (function() {
14475
13491
  * @param {Object} headers - Optional additional headers
14476
13492
  * @returns {Object} - API response parsed from JSON
14477
13493
  */
14478
- makeRequest(url) {
13494
+ async makeRequest(url) {
14479
13495
  console.log(`LinkedIn Ads API Request URL:`, url);
14480
- OAuthUtils.getAccessToken({
13496
+ await OAuthUtils.getAccessToken({
14481
13497
  config: this.config,
14482
13498
  tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
14483
13499
  formData: {
@@ -14492,8 +13508,9 @@ const LinkedInAds = (function() {
14492
13508
  "X-RestLi-Protocol-Version": "2.0.0"
14493
13509
  };
14494
13510
  const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
14495
- const response = EnvironmentAdapter3.fetch(authUrl, { headers });
14496
- const result = JSON.parse(response.getContentText());
13511
+ const response = await HttpUtils3.fetch(authUrl, { headers });
13512
+ const text = await response.getContentText();
13513
+ const result = JSON.parse(text);
14497
13514
  return result;
14498
13515
  }
14499
13516
  /**
@@ -14502,7 +13519,7 @@ const LinkedInAds = (function() {
14502
13519
  * @param {Object} headers - Optional additional headers
14503
13520
  * @returns {Array} - Combined array of results from all pages
14504
13521
  */
14505
- fetchWithPagination(baseUrl) {
13522
+ async fetchWithPagination(baseUrl) {
14506
13523
  let allResults = [];
14507
13524
  let pageToken = null;
14508
13525
  do {
@@ -14510,7 +13527,7 @@ const LinkedInAds = (function() {
14510
13527
  if (pageToken) {
14511
13528
  pageUrl += `${pageUrl.includes("?") ? "&" : "?"}pageToken=${encodeURIComponent(pageToken)}`;
14512
13529
  }
14513
- const res = this.makeRequest(pageUrl);
13530
+ const res = await this.makeRequest(pageUrl);
14514
13531
  const elements = res.elements || [];
14515
13532
  allResults = allResults.concat(elements);
14516
13533
  const metadata = res.metadata || {};
@@ -14520,7 +13537,7 @@ const LinkedInAds = (function() {
14520
13537
  }
14521
13538
  };
14522
13539
  var LinkedInAdsConnector = class LinkedInAdsConnector extends AbstractConnector3 {
14523
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
13540
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
14524
13541
  super(config, source, null, runConfig);
14525
13542
  this.storageName = storageName;
14526
13543
  }
@@ -14528,11 +13545,11 @@ const LinkedInAds = (function() {
14528
13545
  * Main method - entry point for the import process
14529
13546
  * Processes all nodes defined in the fields configuration
14530
13547
  */
14531
- startImportProcess() {
13548
+ async startImportProcess() {
14532
13549
  const urns = FormatUtils.parseIds(this.config.AccountURNs.value, { prefix: "urn:li:sponsoredAccount:" });
14533
13550
  const dataSources = FormatUtils.parseFields(this.config.Fields.value);
14534
13551
  for (const nodeName in dataSources) {
14535
- this.processNode({
13552
+ await this.processNode({
14536
13553
  nodeName,
14537
13554
  urns,
14538
13555
  fields: dataSources[nodeName] || []
@@ -14546,13 +13563,13 @@ const LinkedInAds = (function() {
14546
13563
  * @param {Array} options.urns - URNs to process
14547
13564
  * @param {Array} options.fields - Fields to fetch
14548
13565
  */
14549
- processNode({ nodeName, urns, fields }) {
13566
+ async processNode({ nodeName, urns, fields }) {
14550
13567
  const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
14551
13568
  const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
14552
13569
  if (isTimeSeriesNode && !dateInfo) {
14553
13570
  return;
14554
13571
  }
14555
- this.fetchAndSaveData({
13572
+ await this.fetchAndSaveData({
14556
13573
  nodeName,
14557
13574
  urns,
14558
13575
  fields,
@@ -14573,16 +13590,17 @@ const LinkedInAds = (function() {
14573
13590
  * @param {string} [options.startDate] - Start date for time series data
14574
13591
  * @param {string} [options.endDate] - End date for time series data
14575
13592
  */
14576
- fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
13593
+ async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
14577
13594
  var _a;
14578
13595
  for (const urn of urns) {
14579
13596
  console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
14580
13597
  const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
14581
- const data = this.source.fetchData(nodeName, urn, params);
13598
+ const data = await this.source.fetchData(nodeName, urn, params);
14582
13599
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
14583
13600
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
14584
13601
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
14585
- this.getStorageByNode(nodeName).saveData(preparedData);
13602
+ const storage = await this.getStorageByNode(nodeName);
13603
+ await storage.saveData(preparedData);
14586
13604
  }
14587
13605
  }
14588
13606
  }
@@ -14591,7 +13609,7 @@ const LinkedInAds = (function() {
14591
13609
  * @param {string} nodeName - Name of the node
14592
13610
  * @returns {Object} - Storage instance
14593
13611
  */
14594
- getStorageByNode(nodeName) {
13612
+ async getStorageByNode(nodeName) {
14595
13613
  if (!("storages" in this)) {
14596
13614
  this.storages = {};
14597
13615
  }
@@ -14609,6 +13627,7 @@ const LinkedInAds = (function() {
14609
13627
  this.source.fieldsSchema[nodeName]["fields"],
14610
13628
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
14611
13629
  );
13630
+ await this.storages[nodeName].init();
14612
13631
  }
14613
13632
  return this.storages[nodeName];
14614
13633
  }
@@ -14645,7 +13664,7 @@ const LinkedInAds = (function() {
14645
13664
  };
14646
13665
  })();
14647
13666
  const GoogleAds = (function() {
14648
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
13667
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
14649
13668
  var keywordStatsFields = {
14650
13669
  "keyword_id": {
14651
13670
  "description": "Keyword Criterion ID",
@@ -15546,7 +14565,7 @@ const GoogleAds = (function() {
15546
14565
  * Get access token based on authentication type
15547
14566
  * Supports OAuth2 and Service Account authentication
15548
14567
  */
15549
- getAccessToken() {
14568
+ async getAccessToken() {
15550
14569
  var _a;
15551
14570
  if (this.accessToken && this.tokenExpiryTime && Date.now() < this.tokenExpiryTime) {
15552
14571
  return this.accessToken;
@@ -15559,7 +14578,7 @@ const GoogleAds = (function() {
15559
14578
  let accessToken;
15560
14579
  try {
15561
14580
  if (authType === "oauth2") {
15562
- accessToken = OAuthUtils.getAccessToken({
14581
+ accessToken = await OAuthUtils.getAccessToken({
15563
14582
  config: this.config,
15564
14583
  tokenUrl: "https://oauth2.googleapis.com/token",
15565
14584
  formData: {
@@ -15570,7 +14589,7 @@ const GoogleAds = (function() {
15570
14589
  }
15571
14590
  });
15572
14591
  } else if (authType === "service_account") {
15573
- accessToken = OAuthUtils.getServiceAccountToken({
14592
+ accessToken = await OAuthUtils.getServiceAccountToken({
15574
14593
  config: this.config,
15575
14594
  tokenUrl: "https://oauth2.googleapis.com/token",
15576
14595
  serviceAccountKeyJson: authConfig.ServiceAccountKey.value,
@@ -15597,11 +14616,12 @@ const GoogleAds = (function() {
15597
14616
  * @param {Date} [options.startDate] - Start date for time series data
15598
14617
  * @returns {Array<Object>} - Fetched data
15599
14618
  */
15600
- fetchData(nodeName, customerId, options) {
14619
+ async fetchData(nodeName, customerId, options) {
15601
14620
  console.log("Fetching data from Google Ads API for customer:", customerId);
15602
14621
  const { fields, startDate } = options;
15603
14622
  const query = this._buildQuery({ nodeName, fields, startDate });
15604
- return this.makeRequest({ customerId, query, nodeName, fields });
14623
+ const response = await this.makeRequest({ customerId, query, nodeName, fields });
14624
+ return await response;
15605
14625
  }
15606
14626
  /**
15607
14627
  * Convert field names to API field names
@@ -15651,7 +14671,7 @@ const GoogleAds = (function() {
15651
14671
  const resourceName = this._getResourceName(nodeName);
15652
14672
  let query = `SELECT ${apiFields.join(", ")} FROM ${resourceName}`;
15653
14673
  if (startDate && this.fieldsSchema[nodeName].isTimeSeries) {
15654
- const formattedDate = EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd");
14674
+ const formattedDate = DateUtils3.formatDate(startDate);
15655
14675
  query += ` WHERE segments.date = '${formattedDate}'`;
15656
14676
  }
15657
14677
  return query;
@@ -15665,9 +14685,9 @@ const GoogleAds = (function() {
15665
14685
  * @param {Array<string>} options.fields - Fields that were requested
15666
14686
  * @returns {Array<Object>} - API response data
15667
14687
  */
15668
- makeRequest({ customerId, query, nodeName, fields }) {
14688
+ async makeRequest({ customerId, query, nodeName, fields }) {
15669
14689
  var _a, _b;
15670
- const accessToken = this.getAccessToken();
14690
+ const accessToken = await this.getAccessToken();
15671
14691
  const url = `https://googleads.googleapis.com/v21/customers/${customerId}/googleAds:search`;
15672
14692
  console.log(`Google Ads API Request URL: ${url}`);
15673
14693
  console.log(`GAQL Query: ${query}`);
@@ -15691,8 +14711,9 @@ const GoogleAds = (function() {
15691
14711
  body: JSON.stringify(requestBody),
15692
14712
  muteHttpExceptions: true
15693
14713
  };
15694
- const response = this.urlFetchWithRetry(url, options);
15695
- const jsonData = JSON.parse(response.getContentText());
14714
+ const response = await this.urlFetchWithRetry(url, options);
14715
+ const text = await response.getContentText();
14716
+ const jsonData = JSON.parse(text);
15696
14717
  if (jsonData.error) {
15697
14718
  throw new Error(`Google Ads API error: ${jsonData.error.message}`);
15698
14719
  }
@@ -15747,7 +14768,7 @@ const GoogleAds = (function() {
15747
14768
  }
15748
14769
  };
15749
14770
  var GoogleAdsConnector = class GoogleAdsConnector extends AbstractConnector3 {
15750
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
14771
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
15751
14772
  super(config, source, null, runConfig);
15752
14773
  this.storageName = storageName;
15753
14774
  }
@@ -15755,11 +14776,11 @@ const GoogleAds = (function() {
15755
14776
  * Main method - entry point for the import process
15756
14777
  * Processes all nodes defined in the fields configuration
15757
14778
  */
15758
- startImportProcess() {
14779
+ async startImportProcess() {
15759
14780
  const customerIds = FormatUtils.parseIds(this.config.CustomerId.value, { stripCharacters: "-" });
15760
14781
  const fields = FormatUtils.parseFields(this.config.Fields.value);
15761
14782
  for (const nodeName in fields) {
15762
- this.processNode({
14783
+ await this.processNode({
15763
14784
  nodeName,
15764
14785
  customerIds,
15765
14786
  fields: fields[nodeName] || []
@@ -15773,16 +14794,16 @@ const GoogleAds = (function() {
15773
14794
  * @param {Array<string>} options.customerIds - Array of customer IDs to process
15774
14795
  * @param {Array<string>} options.fields - Array of fields to fetch
15775
14796
  */
15776
- processNode({ nodeName, customerIds, fields }) {
14797
+ async processNode({ nodeName, customerIds, fields }) {
15777
14798
  for (const customerId of customerIds) {
15778
14799
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
15779
- this.processTimeSeriesNode({
14800
+ await this.processTimeSeriesNode({
15780
14801
  nodeName,
15781
14802
  customerId,
15782
14803
  fields
15783
14804
  });
15784
14805
  } else {
15785
- this.processCatalogNode({
14806
+ await this.processCatalogNode({
15786
14807
  nodeName,
15787
14808
  customerId,
15788
14809
  fields
@@ -15797,7 +14818,7 @@ const GoogleAds = (function() {
15797
14818
  * @param {string} options.customerId - Customer ID
15798
14819
  * @param {Array<string>} options.fields - Array of fields to fetch
15799
14820
  */
15800
- processTimeSeriesNode({ nodeName, customerId, fields }) {
14821
+ async processTimeSeriesNode({ nodeName, customerId, fields }) {
15801
14822
  var _a;
15802
14823
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
15803
14824
  if (daysToFetch <= 0) {
@@ -15807,12 +14828,13 @@ const GoogleAds = (function() {
15807
14828
  for (let i = 0; i < daysToFetch; i++) {
15808
14829
  const currentDate = new Date(startDate);
15809
14830
  currentDate.setDate(currentDate.getDate() + i);
15810
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
15811
- const data = this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
14831
+ const formattedDate = DateUtils3.formatDate(currentDate);
14832
+ const data = await this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
15812
14833
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId} on ${formattedDate}` : `ℹ️ No records have been fetched`);
15813
14834
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
15814
14835
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
15815
- this.getStorageByNode(nodeName).saveData(preparedData);
14836
+ const storage = await this.getStorageByNode(nodeName);
14837
+ await storage.saveData(preparedData);
15816
14838
  }
15817
14839
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
15818
14840
  this.config.updateLastRequstedDate(currentDate);
@@ -15826,13 +14848,14 @@ const GoogleAds = (function() {
15826
14848
  * @param {string} options.customerId - Customer ID
15827
14849
  * @param {Array<string>} options.fields - Array of fields to fetch
15828
14850
  */
15829
- processCatalogNode({ nodeName, customerId, fields }) {
14851
+ async processCatalogNode({ nodeName, customerId, fields }) {
15830
14852
  var _a;
15831
- const data = this.source.fetchData(nodeName, customerId, { fields });
14853
+ const data = await this.source.fetchData(nodeName, customerId, { fields });
15832
14854
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId}` : `ℹ️ No records have been fetched`);
15833
14855
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
15834
14856
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
15835
- this.getStorageByNode(nodeName).saveData(preparedData);
14857
+ const storage = await this.getStorageByNode(nodeName);
14858
+ await storage.saveData(preparedData);
15836
14859
  }
15837
14860
  }
15838
14861
  /**
@@ -15840,7 +14863,7 @@ const GoogleAds = (function() {
15840
14863
  * @param {string} nodeName - Name of the node
15841
14864
  * @returns {Object} - Storage instance
15842
14865
  */
15843
- getStorageByNode(nodeName) {
14866
+ async getStorageByNode(nodeName) {
15844
14867
  if (!("storages" in this)) {
15845
14868
  this.storages = {};
15846
14869
  }
@@ -15858,6 +14881,7 @@ const GoogleAds = (function() {
15858
14881
  this.source.fieldsSchema[nodeName].fields,
15859
14882
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
15860
14883
  );
14884
+ await this.storages[nodeName].init();
15861
14885
  }
15862
14886
  return this.storages[nodeName];
15863
14887
  }
@@ -15874,7 +14898,7 @@ const GoogleAds = (function() {
15874
14898
  };
15875
14899
  })();
15876
14900
  const GitHub = (function() {
15877
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
14901
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
15878
14902
  var repositoryStatsFields = {
15879
14903
  date: {
15880
14904
  type: "date",
@@ -16187,16 +15211,16 @@ const GitHub = (function() {
16187
15211
  * @param {Array<string>} opts.fields
16188
15212
  * @returns {Array<Object>}
16189
15213
  */
16190
- fetchData({ nodeName, fields = [] }) {
15214
+ async fetchData({ nodeName, fields = [] }) {
16191
15215
  switch (nodeName) {
16192
15216
  case "repository":
16193
- const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
15217
+ const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
16194
15218
  return this._filterBySchema({ items: [repoData], nodeName, fields });
16195
15219
  case "contributors":
16196
- const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
15220
+ const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
16197
15221
  return this._filterBySchema({ items: contribData, nodeName, fields });
16198
15222
  case "repositoryStats":
16199
- return this._fetchRepositoryStats({ nodeName, fields });
15223
+ return await this._fetchRepositoryStats({ nodeName, fields });
16200
15224
  default:
16201
15225
  throw new Error(`Unknown node: ${nodeName}`);
16202
15226
  }
@@ -16208,9 +15232,9 @@ const GitHub = (function() {
16208
15232
  * @param {Array<string>} options.fields - Array of fields to fetch
16209
15233
  * @returns {Array} Array of repository statistics data
16210
15234
  */
16211
- _fetchRepositoryStats({ nodeName, fields }) {
16212
- const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
16213
- const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
15235
+ async _fetchRepositoryStats({ nodeName, fields }) {
15236
+ const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
15237
+ const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
16214
15238
  return this._filterBySchema({
16215
15239
  items: [{
16216
15240
  "date": new Date((/* @__PURE__ */ new Date()).setHours(0, 0, 0, 0)),
@@ -16227,30 +15251,32 @@ const GitHub = (function() {
16227
15251
  * @param {string} options.endpoint - API endpoint path (e.g., "repos/owner/repo")
16228
15252
  * @returns {Object} - API response parsed from JSON
16229
15253
  */
16230
- makeRequest({ endpoint }) {
16231
- const baseUrl = "https://api.github.com/";
16232
- const url = `${baseUrl}${endpoint}`;
16233
- const response = EnvironmentAdapter3.fetch(url, {
16234
- "method": "get",
16235
- "muteHttpExceptions": true,
16236
- "headers": {
16237
- "Accept": "application/vnd.github+json",
16238
- "Authorization": `Bearer ${this.config.AccessToken.value}`,
16239
- "User-Agent": "owox"
15254
+ async makeRequest({ endpoint }) {
15255
+ try {
15256
+ const baseUrl = "https://api.github.com/";
15257
+ const url = `${baseUrl}${endpoint}`;
15258
+ const response = await HttpUtils3.fetch(url, {
15259
+ "method": "get",
15260
+ "muteHttpExceptions": true,
15261
+ "headers": {
15262
+ "Accept": "application/vnd.github+json",
15263
+ "Authorization": `Bearer ${this.config.AccessToken.value}`,
15264
+ "User-Agent": "owox"
15265
+ }
15266
+ });
15267
+ const text = await response.getContentText();
15268
+ const result = JSON.parse(text);
15269
+ if (result && result.message === "Not Found") {
15270
+ throw new Error(
15271
+ "The repository was not found. The repository name should be in the format: owner/repo"
15272
+ );
16240
15273
  }
16241
- });
16242
- const result = JSON.parse(response.getContentText());
16243
- if (result && result.message === "Not Found") {
16244
- throw new Error(
16245
- "The repository was not found. The repository name should be in the format: owner/repo"
16246
- );
15274
+ return result;
15275
+ } catch (error) {
15276
+ this.config.logMessage(`Error: ${error.message}`);
15277
+ console.error(error.stack);
15278
+ throw error;
16247
15279
  }
16248
- return result;
16249
- }
16250
- catch(error) {
16251
- this.config.logMessage(`Error: ${error.message}`);
16252
- console.error(error.stack);
16253
- throw error;
16254
15280
  }
16255
15281
  /**
16256
15282
  * Keep only requestedFields plus any schema-required keys.
@@ -16276,7 +15302,7 @@ const GitHub = (function() {
16276
15302
  }
16277
15303
  };
16278
15304
  var GitHubConnector = class GitHubConnector extends AbstractConnector3 {
16279
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
15305
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
16280
15306
  super(config, source, null, runConfig);
16281
15307
  this.storageName = storageName;
16282
15308
  }
@@ -16284,10 +15310,10 @@ const GitHub = (function() {
16284
15310
  * Main method - entry point for the import process
16285
15311
  * Processes all nodes defined in the fields configuration
16286
15312
  */
16287
- startImportProcess() {
15313
+ async startImportProcess() {
16288
15314
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
16289
15315
  for (const nodeName in fields) {
16290
- this.processNode({
15316
+ await this.processNode({
16291
15317
  nodeName,
16292
15318
  fields: fields[nodeName] || []
16293
15319
  });
@@ -16299,11 +15325,11 @@ const GitHub = (function() {
16299
15325
  * @param {string} options.nodeName - Name of the node to process
16300
15326
  * @param {Array<string>} options.fields - Array of fields to fetch
16301
15327
  */
16302
- processNode({ nodeName, fields }) {
15328
+ async processNode({ nodeName, fields }) {
16303
15329
  if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
16304
- this.processTimeSeriesNode({ nodeName, fields });
15330
+ await this.processTimeSeriesNode({ nodeName, fields });
16305
15331
  } else {
16306
- this.processCatalogNode({ nodeName, fields });
15332
+ await this.processCatalogNode({ nodeName, fields });
16307
15333
  }
16308
15334
  }
16309
15335
  /**
@@ -16313,7 +15339,7 @@ const GitHub = (function() {
16313
15339
  * @param {Array<string>} options.fields - Array of fields to fetch
16314
15340
  * @param {Object} options.storage - Storage instance
16315
15341
  */
16316
- processTimeSeriesNode({ nodeName, fields }) {
15342
+ async processTimeSeriesNode({ nodeName, fields }) {
16317
15343
  console.log(`Time series node processing not implemented for ${nodeName}`);
16318
15344
  }
16319
15345
  /**
@@ -16323,13 +15349,14 @@ const GitHub = (function() {
16323
15349
  * @param {Array<string>} options.fields - Array of fields to fetch
16324
15350
  * @param {Object} options.storage - Storage instance
16325
15351
  */
16326
- processCatalogNode({ nodeName, fields }) {
15352
+ async processCatalogNode({ nodeName, fields }) {
16327
15353
  var _a;
16328
- const data = this.source.fetchData({ nodeName, fields });
15354
+ const data = await this.source.fetchData({ nodeName, fields });
16329
15355
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched` : `No records have been fetched`);
16330
15356
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
16331
15357
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
16332
- this.getStorageByNode(nodeName).saveData(preparedData);
15358
+ const storage = await this.getStorageByNode(nodeName);
15359
+ await storage.saveData(preparedData);
16333
15360
  }
16334
15361
  }
16335
15362
  /**
@@ -16337,7 +15364,7 @@ const GitHub = (function() {
16337
15364
  * @param {string} nodeName - Name of the node
16338
15365
  * @returns {Object} Storage instance
16339
15366
  */
16340
- getStorageByNode(nodeName) {
15367
+ async getStorageByNode(nodeName) {
16341
15368
  if (!("storages" in this)) {
16342
15369
  this.storages = {};
16343
15370
  }
@@ -16355,6 +15382,7 @@ const GitHub = (function() {
16355
15382
  this.source.fieldsSchema[nodeName].fields,
16356
15383
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
16357
15384
  );
15385
+ await this.storages[nodeName].init();
16358
15386
  }
16359
15387
  return this.storages[nodeName];
16360
15388
  }
@@ -16371,7 +15399,7 @@ const GitHub = (function() {
16371
15399
  };
16372
15400
  })();
16373
15401
  const FacebookMarketing = (function() {
16374
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
15402
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
16375
15403
  var adGroupFields = {
16376
15404
  "id": {
16377
15405
  "description": "The ID of this ad.",
@@ -20876,12 +19904,12 @@ const FacebookMarketing = (function() {
20876
19904
  @return data array
20877
19905
 
20878
19906
  */
20879
- fetchData(nodeName, accountId, fields, startDate = null) {
19907
+ async fetchData(nodeName, accountId, fields, startDate = null) {
20880
19908
  let url = "https://graph.facebook.com/v23.0/";
20881
19909
  let formattedDate = null;
20882
19910
  let timeRange = null;
20883
19911
  if (startDate) {
20884
- formattedDate = EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd");
19912
+ formattedDate = DateUtils3.formatDate(startDate);
20885
19913
  timeRange = encodeURIComponent(JSON.stringify({ since: formattedDate, until: formattedDate }));
20886
19914
  }
20887
19915
  switch (nodeName) {
@@ -20905,7 +19933,7 @@ const FacebookMarketing = (function() {
20905
19933
  case "ad-account/insights-by-region":
20906
19934
  case "ad-account/insights-by-product-id":
20907
19935
  case "ad-account/insights-by-age-and-gender":
20908
- return this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
19936
+ return await this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
20909
19937
  case "ad-group":
20910
19938
  url += `act_${accountId}/ads?fields=${this._buildFieldsString({ nodeName, fields })}&limit=${this.fieldsSchema[nodeName].limit}`;
20911
19939
  break;
@@ -20914,7 +19942,7 @@ const FacebookMarketing = (function() {
20914
19942
  }
20915
19943
  console.log(`Facebook API URL:`, url);
20916
19944
  url += `&access_token=${this.config.AccessToken.value}`;
20917
- return this._fetchPaginatedData(url, nodeName, fields);
19945
+ return await this._fetchPaginatedData(url, nodeName, fields);
20918
19946
  }
20919
19947
  //---- castRecordFields -------------------------------------------------
20920
19948
  /**
@@ -20976,7 +20004,7 @@ const FacebookMarketing = (function() {
20976
20004
  * @return {Array} Processed insights data
20977
20005
  * @private
20978
20006
  */
20979
- _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
20007
+ async _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
20980
20008
  const breakdowns = this.fieldsSchema[nodeName].breakdowns || [];
20981
20009
  const regularFields = this._prepareFields({ nodeName, fields, breakdowns });
20982
20010
  const requestUrl = this._buildInsightsUrl({
@@ -20987,7 +20015,7 @@ const FacebookMarketing = (function() {
20987
20015
  nodeName,
20988
20016
  url
20989
20017
  });
20990
- const allData = this._fetchPaginatedData(requestUrl, nodeName, fields);
20018
+ const allData = await this._fetchPaginatedData(requestUrl, nodeName, fields);
20991
20019
  if (this.config.ProcessShortLinks.value && allData.length > 0 && allData.some((record) => record.link_url_asset)) {
20992
20020
  return processShortLinks(allData, {
20993
20021
  shortLinkField: "link_url_asset",
@@ -21128,12 +20156,13 @@ const FacebookMarketing = (function() {
21128
20156
  * @return {Array} All fetched data
21129
20157
  * @private
21130
20158
  */
21131
- _fetchPaginatedData(initialUrl, nodeName, fields) {
20159
+ async _fetchPaginatedData(initialUrl, nodeName, fields) {
21132
20160
  var allData = [];
21133
20161
  var nextPageURL = initialUrl;
21134
20162
  while (nextPageURL) {
21135
- var response = this.urlFetchWithRetry(nextPageURL);
21136
- var jsonData = JSON.parse(response.getContentText());
20163
+ var response = await this.urlFetchWithRetry(nextPageURL);
20164
+ var text = await response.getContentText();
20165
+ var jsonData = JSON.parse(text);
21137
20166
  if ("data" in jsonData) {
21138
20167
  nextPageURL = jsonData.paging ? jsonData.paging.next : null;
21139
20168
  jsonData.data.forEach((record, index) => {
@@ -21156,12 +20185,12 @@ const FacebookMarketing = (function() {
21156
20185
  };
21157
20186
  var FacebookMarketingConnector = class FacebookMarketingConnector extends AbstractConnector3 {
21158
20187
  // ---- constructor ------------------------------------
21159
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
20188
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
21160
20189
  super(config, source, null, runConfig);
21161
20190
  this.storageName = storageName;
21162
20191
  }
21163
20192
  //---- startImportProcess -------------------------------------------------
21164
- startImportProcess() {
20193
+ async startImportProcess() {
21165
20194
  let accountsIds = String(this.config.AccoundIDs.value).split(/[,;]\s*/);
21166
20195
  let fields = this.config.Fields.value.split(", ").reduce((acc, pair) => {
21167
20196
  let [key, value] = pair.split(" ");
@@ -21173,7 +20202,7 @@ const FacebookMarketing = (function() {
21173
20202
  if (nodeName in this.source.fieldsSchema && this.source.fieldsSchema[nodeName].isTimeSeries) {
21174
20203
  timeSeriesNodes[nodeName] = fields[nodeName];
21175
20204
  } else {
21176
- this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
20205
+ await this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
21177
20206
  }
21178
20207
  }
21179
20208
  if (Object.keys(timeSeriesNodes).length > 0) {
@@ -21181,27 +20210,28 @@ const FacebookMarketing = (function() {
21181
20210
  let daysToFetch = null;
21182
20211
  [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
21183
20212
  if (daysToFetch > 0) {
21184
- this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
20213
+ await this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
21185
20214
  }
21186
20215
  }
21187
20216
  }
21188
20217
  //---- startImportProcessOfCatalogData -------------------------------------------------
21189
20218
  /*
21190
20219
 
21191
- Imports catalog (not time seriesed) data
20220
+ Imports catalog (not time seriesed) data
21192
20221
 
21193
20222
  @param nodeName string Node name
21194
20223
  @param accountsIds array list of account ids
21195
- @param fields array list of fields
20224
+ @param fields array list of fields
21196
20225
 
21197
20226
  */
21198
- startImportProcessOfCatalogData(nodeName, accountIds, fields) {
20227
+ async startImportProcessOfCatalogData(nodeName, accountIds, fields) {
21199
20228
  var _a;
21200
20229
  for (var i in accountIds) {
21201
20230
  let accountId = accountIds[i];
21202
- let data = this.source.fetchData(nodeName, accountId, fields);
20231
+ let data = await this.source.fetchData(nodeName, accountId, fields);
21203
20232
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
21204
- this.getStorageByNode(nodeName).saveData(data);
20233
+ const storage = await this.getStorageByNode(nodeName);
20234
+ await storage.saveData(data);
21205
20235
  }
21206
20236
  data.length && this.config.logMessage(`${data.length} rows of ${nodeName} were fetched for account ${accountId}`);
21207
20237
  }
@@ -21209,23 +20239,24 @@ const FacebookMarketing = (function() {
21209
20239
  //---- startImportProcessOfTimeSeriesData -------------------------------------------------
21210
20240
  /*
21211
20241
 
21212
- Imports time series (not catalog) data
20242
+ Imports time series (not catalog) data
21213
20243
 
21214
20244
  @param accountsIds (array) list of account ids
21215
20245
  @param timeSeriesNodes (object) of properties, each is array of fields
21216
- @param startDate (Data) start date
20246
+ @param startDate (Data) start date
21217
20247
  @param daysToFetch (integer) days to import
21218
20248
 
21219
20249
  */
21220
- startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
20250
+ async startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
21221
20251
  var _a;
21222
20252
  for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
21223
20253
  for (let accountId of accountsIds) {
21224
20254
  for (var nodeName in timeSeriesNodes) {
21225
- this.config.logMessage(`Start importing data for ${EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd")}: ${accountId}/${nodeName}`);
21226
- let data = this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
20255
+ this.config.logMessage(`Start importing data for ${DateUtils3.formatDate(startDate)}: ${accountId}/${nodeName}`);
20256
+ let data = await this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
21227
20257
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
21228
- this.getStorageByNode(nodeName).saveData(data);
20258
+ const storage = await this.getStorageByNode(nodeName);
20259
+ await storage.saveData(data);
21229
20260
  }
21230
20261
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
21231
20262
  }
@@ -21245,7 +20276,7 @@ const FacebookMarketing = (function() {
21245
20276
  * @return AbstractStorage
21246
20277
  *
21247
20278
  */
21248
- getStorageByNode(nodeName) {
20279
+ async getStorageByNode(nodeName) {
21249
20280
  if (!("storages" in this)) {
21250
20281
  this.storages = {};
21251
20282
  }
@@ -21263,6 +20294,7 @@ const FacebookMarketing = (function() {
21263
20294
  this.source.fieldsSchema[nodeName]["fields"],
21264
20295
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
21265
20296
  );
20297
+ await this.storages[nodeName].init();
21266
20298
  }
21267
20299
  return this.storages[nodeName];
21268
20300
  }
@@ -21279,7 +20311,7 @@ const FacebookMarketing = (function() {
21279
20311
  };
21280
20312
  })();
21281
20313
  const CriteoAds = (function() {
21282
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
20314
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
21283
20315
  var CriteoAdsHelper = {
21284
20316
  /**
21285
20317
  * Parse fields string into a structured object
@@ -22126,10 +21158,10 @@ const CriteoAds = (function() {
22126
21158
  * @param {Date} opts.date
22127
21159
  * @returns {Array<Object>}
22128
21160
  */
22129
- fetchData({ nodeName, accountId, fields = [], date }) {
21161
+ async fetchData({ nodeName, accountId, fields = [], date }) {
22130
21162
  switch (nodeName) {
22131
21163
  case "statistics":
22132
- return this._fetchStatistics({ accountId, fields, date });
21164
+ return await this._fetchStatistics({ accountId, fields, date });
22133
21165
  default:
22134
21166
  throw new Error(`Unknown node: ${nodeName}`);
22135
21167
  }
@@ -22143,16 +21175,17 @@ const CriteoAds = (function() {
22143
21175
  * @returns {Array<Object>} - Parsed and enriched data
22144
21176
  * @private
22145
21177
  */
22146
- _fetchStatistics({ accountId, fields, date }) {
21178
+ async _fetchStatistics({ accountId, fields, date }) {
22147
21179
  const uniqueKeys = this.fieldsSchema.statistics.uniqueKeys || [];
22148
21180
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
22149
21181
  if (missingKeys.length > 0) {
22150
21182
  throw new Error(`Missing required unique fields for endpoint 'statistics'. Missing fields: ${missingKeys.join(", ")}`);
22151
21183
  }
22152
- this.getAccessToken();
21184
+ await this.getAccessToken();
22153
21185
  const requestBody = this._buildStatisticsRequestBody({ accountId, fields, date });
22154
- const response = this._makeApiRequest(requestBody);
22155
- const jsonObject = JSON.parse(response.getContentText());
21186
+ const response = await this._makeApiRequest(requestBody);
21187
+ const text = await response.getContentText();
21188
+ const jsonObject = JSON.parse(text);
22156
21189
  return this.parseApiResponse({ apiResponse: jsonObject, date, accountId, fields });
22157
21190
  }
22158
21191
  /**
@@ -22190,7 +21223,7 @@ const CriteoAds = (function() {
22190
21223
  * @returns {Object} - HTTP response
22191
21224
  * @private
22192
21225
  */
22193
- _makeApiRequest(requestBody) {
21226
+ async _makeApiRequest(requestBody) {
22194
21227
  const apiVersion = "2025-07";
22195
21228
  const apiUrl = `https://api.criteo.com/${apiVersion}/statistics/report`;
22196
21229
  const options = {
@@ -22204,19 +21237,20 @@ const CriteoAds = (function() {
22204
21237
  body: JSON.stringify(requestBody)
22205
21238
  // TODO: body is for Node.js; refactor to centralize JSON option creation
22206
21239
  };
22207
- const response = this.urlFetchWithRetry(apiUrl, options);
21240
+ const response = await this.urlFetchWithRetry(apiUrl, options);
22208
21241
  const responseCode = response.getResponseCode();
22209
21242
  if (responseCode === HTTP_STATUS2.OK) {
22210
21243
  return response;
22211
21244
  } else {
22212
- throw new Error(`API Error (${responseCode}): ${response.getContentText()}`);
21245
+ const text = await response.getContentText();
21246
+ throw new Error(`API Error (${responseCode}): ${text}`);
22213
21247
  }
22214
21248
  }
22215
21249
  /**
22216
21250
  * Get access token from API
22217
21251
  * Docs: https://developers.criteo.com/marketing-solutions/docs/authorization-code-setup
22218
21252
  */
22219
- getAccessToken() {
21253
+ async getAccessToken() {
22220
21254
  var _a;
22221
21255
  if ((_a = this.config.AccessToken) == null ? void 0 : _a.value) {
22222
21256
  return;
@@ -22239,8 +21273,9 @@ const CriteoAds = (function() {
22239
21273
  muteHttpExceptions: true
22240
21274
  };
22241
21275
  try {
22242
- const response = this.urlFetchWithRetry(tokenUrl, options);
22243
- const responseData = JSON.parse(response.getContentText());
21276
+ const response = await this.urlFetchWithRetry(tokenUrl, options);
21277
+ const text = await response.getContentText();
21278
+ const responseData = JSON.parse(text);
22244
21279
  const accessToken = responseData["access_token"];
22245
21280
  this.config.AccessToken = {
22246
21281
  value: accessToken
@@ -22288,20 +21323,20 @@ const CriteoAds = (function() {
22288
21323
  }
22289
21324
  };
22290
21325
  var CriteoAdsConnector = class CriteoAdsConnector extends AbstractConnector3 {
22291
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
21326
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
22292
21327
  super(config, source, null, runConfig);
22293
21328
  this.storageName = storageName;
22294
21329
  }
22295
21330
  /**
22296
21331
  * Main method - entry point for the import process
22297
21332
  */
22298
- startImportProcess() {
21333
+ async startImportProcess() {
22299
21334
  var _a, _b;
22300
21335
  const fields = CriteoAdsHelper.parseFields(((_a = this.config.Fields) == null ? void 0 : _a.value) || "");
22301
21336
  const advertiserIds = CriteoAdsHelper.parseAdvertiserIds(((_b = this.config.AdvertiserIDs) == null ? void 0 : _b.value) || "");
22302
21337
  for (const advertiserId of advertiserIds) {
22303
21338
  for (const nodeName in fields) {
22304
- this.processNode({
21339
+ await this.processNode({
22305
21340
  nodeName,
22306
21341
  advertiserId,
22307
21342
  fields: fields[nodeName] || []
@@ -22316,8 +21351,8 @@ const CriteoAds = (function() {
22316
21351
  * @param {string} options.advertiserId - Advertiser ID
22317
21352
  * @param {Array<string>} options.fields - Array of fields to fetch
22318
21353
  */
22319
- processNode({ nodeName, advertiserId, fields }) {
22320
- this.processTimeSeriesNode({
21354
+ async processNode({ nodeName, advertiserId, fields }) {
21355
+ await this.processTimeSeriesNode({
22321
21356
  nodeName,
22322
21357
  advertiserId,
22323
21358
  fields
@@ -22331,7 +21366,7 @@ const CriteoAds = (function() {
22331
21366
  * @param {Array<string>} options.fields - Array of fields to fetch
22332
21367
  * @param {Object} options.storage - Storage instance
22333
21368
  */
22334
- processTimeSeriesNode({ nodeName, advertiserId, fields }) {
21369
+ async processTimeSeriesNode({ nodeName, advertiserId, fields }) {
22335
21370
  var _a;
22336
21371
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
22337
21372
  if (daysToFetch <= 0) {
@@ -22341,8 +21376,8 @@ const CriteoAds = (function() {
22341
21376
  for (let i = 0; i < daysToFetch; i++) {
22342
21377
  const currentDate = new Date(startDate);
22343
21378
  currentDate.setDate(currentDate.getDate() + i);
22344
- const formattedDate = EnvironmentAdapter3.formatDate(currentDate, "UTC", "yyyy-MM-dd");
22345
- const data = this.source.fetchData({
21379
+ const formattedDate = DateUtils3.formatDate(currentDate);
21380
+ const data = await this.source.fetchData({
22346
21381
  nodeName,
22347
21382
  accountId: advertiserId,
22348
21383
  date: currentDate,
@@ -22351,7 +21386,8 @@ const CriteoAds = (function() {
22351
21386
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${advertiserId} on ${formattedDate}` : `No records have been fetched`);
22352
21387
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
22353
21388
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
22354
- this.getStorageByNode(nodeName).saveData(preparedData);
21389
+ const storage = await this.getStorageByNode(nodeName);
21390
+ await storage.saveData(preparedData);
22355
21391
  }
22356
21392
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
22357
21393
  this.config.updateLastRequstedDate(currentDate);
@@ -22363,7 +21399,7 @@ const CriteoAds = (function() {
22363
21399
  * @param {string} nodeName - Name of the node
22364
21400
  * @returns {Object} Storage instance
22365
21401
  */
22366
- getStorageByNode(nodeName) {
21402
+ async getStorageByNode(nodeName) {
22367
21403
  if (!("storages" in this)) {
22368
21404
  this.storages = {};
22369
21405
  }
@@ -22381,6 +21417,7 @@ const CriteoAds = (function() {
22381
21417
  this.source.fieldsSchema[nodeName].fields,
22382
21418
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
22383
21419
  );
21420
+ await this.storages[nodeName].init();
22384
21421
  }
22385
21422
  return this.storages[nodeName];
22386
21423
  }
@@ -22397,7 +21434,7 @@ const CriteoAds = (function() {
22397
21434
  };
22398
21435
  })();
22399
21436
  const BankOfCanada = (function() {
22400
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
21437
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
22401
21438
  var observationsFields = {
22402
21439
  "date": {
22403
21440
  "description": "The date for which the exchange rate was recorded.",
@@ -22475,10 +21512,10 @@ const BankOfCanada = (function() {
22475
21512
  * @param {string} [opts.end_time]
22476
21513
  * @returns {Array<Object>}
22477
21514
  */
22478
- fetchData({ nodeName, fields = [], start_time, end_time }) {
21515
+ async fetchData({ nodeName, fields = [], start_time, end_time }) {
22479
21516
  switch (nodeName) {
22480
21517
  case "observations/group":
22481
- return this._fetchObservations({ fields, start_time, end_time });
21518
+ return await this._fetchObservations({ fields, start_time, end_time });
22482
21519
  default:
22483
21520
  throw new Error(`Unknown node: ${nodeName}`);
22484
21521
  }
@@ -22491,8 +21528,8 @@ const BankOfCanada = (function() {
22491
21528
  * @param {string} opts.end_time
22492
21529
  * @returns {Array<Object>}
22493
21530
  */
22494
- _fetchObservations({ fields, start_time, end_time }) {
22495
- const rates = this.makeRequest({
21531
+ async _fetchObservations({ fields, start_time, end_time }) {
21532
+ const rates = await this.makeRequest({
22496
21533
  endpoint: `observations/group/FX_RATES_DAILY/json?start_date=${start_time}&end_date=${end_time}`
22497
21534
  });
22498
21535
  let data = [];
@@ -22514,13 +21551,13 @@ const BankOfCanada = (function() {
22514
21551
  * @param {string} options.endpoint - API endpoint path (e.g., "observations/group/FX_RATES_DAILY/json")
22515
21552
  * @returns {Object} - API response parsed from JSON
22516
21553
  */
22517
- makeRequest({ endpoint }) {
21554
+ async makeRequest({ endpoint }) {
22518
21555
  const baseUrl = "https://www.bankofcanada.ca/valet/";
22519
21556
  const url = `${baseUrl}${endpoint}`;
22520
21557
  console.log(`Bank of Canada API Request URL:`, url);
22521
- const response = EnvironmentAdapter3.fetch(url, { "method": "get", "muteHttpExceptions": true });
22522
- const result = JSON.parse(response.getContentText());
22523
- return result;
21558
+ const response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
21559
+ const result = await response.getContentText();
21560
+ return JSON.parse(result);
22524
21561
  }
22525
21562
  /**
22526
21563
  * Keep only requestedFields plus any schema-required keys.
@@ -22545,7 +21582,7 @@ const BankOfCanada = (function() {
22545
21582
  }
22546
21583
  };
22547
21584
  var BankOfCanadaConnector = class BankOfCanadaConnector extends AbstractConnector3 {
22548
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
21585
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
22549
21586
  super(config, source, null, runConfig);
22550
21587
  this.storageName = storageName;
22551
21588
  }
@@ -22553,10 +21590,10 @@ const BankOfCanada = (function() {
22553
21590
  * Main method - entry point for the import process
22554
21591
  * Processes all nodes defined in the fields configuration
22555
21592
  */
22556
- startImportProcess() {
21593
+ async startImportProcess() {
22557
21594
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
22558
21595
  for (const nodeName in fields) {
22559
- this.processNode({
21596
+ await this.processNode({
22560
21597
  nodeName,
22561
21598
  fields: fields[nodeName] || []
22562
21599
  });
@@ -22568,11 +21605,11 @@ const BankOfCanada = (function() {
22568
21605
  * @param {string} options.nodeName - Name of the node to process
22569
21606
  * @param {Array<string>} options.fields - Array of fields to fetch
22570
21607
  */
22571
- processNode({ nodeName, fields }) {
21608
+ async processNode({ nodeName, fields }) {
22572
21609
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
22573
- this.processTimeSeriesNode({ nodeName, fields });
21610
+ await this.processTimeSeriesNode({ nodeName, fields });
22574
21611
  } else {
22575
- this.processCatalogNode({ nodeName, fields });
21612
+ await this.processCatalogNode({ nodeName, fields });
22576
21613
  }
22577
21614
  }
22578
21615
  /**
@@ -22582,14 +21619,14 @@ const BankOfCanada = (function() {
22582
21619
  * @param {Array<string>} options.fields - Array of fields to fetch
22583
21620
  * @param {Object} options.storage - Storage instance
22584
21621
  */
22585
- processTimeSeriesNode({ nodeName, fields }) {
21622
+ async processTimeSeriesNode({ nodeName, fields }) {
22586
21623
  var _a;
22587
21624
  const dateRange = this.prepareDateRange();
22588
21625
  if (!dateRange) {
22589
21626
  console.log("No days to fetch for time series data");
22590
21627
  return;
22591
21628
  }
22592
- const data = this.source.fetchData({
21629
+ const data = await this.source.fetchData({
22593
21630
  nodeName,
22594
21631
  start_time: dateRange.startDate,
22595
21632
  end_time: dateRange.endDate,
@@ -22598,7 +21635,8 @@ const BankOfCanada = (function() {
22598
21635
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
22599
21636
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
22600
21637
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
22601
- this.getStorageByNode(nodeName).saveData(preparedData);
21638
+ const storage = await this.getStorageByNode(nodeName);
21639
+ await storage.saveData(preparedData);
22602
21640
  }
22603
21641
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
22604
21642
  this.config.updateLastRequstedDate(new Date(dateRange.endDate));
@@ -22611,7 +21649,7 @@ const BankOfCanada = (function() {
22611
21649
  * @param {Array<string>} options.fields - Array of fields to fetch
22612
21650
  * @param {Object} options.storage - Storage instance
22613
21651
  */
22614
- processCatalogNode({ nodeName, fields }) {
21652
+ async processCatalogNode({ nodeName, fields }) {
22615
21653
  console.log(`Catalog node processing not implemented for ${nodeName}`);
22616
21654
  }
22617
21655
  /**
@@ -22619,7 +21657,7 @@ const BankOfCanada = (function() {
22619
21657
  * @param {string} nodeName - Name of the node
22620
21658
  * @returns {Object} Storage instance
22621
21659
  */
22622
- getStorageByNode(nodeName) {
21660
+ async getStorageByNode(nodeName) {
22623
21661
  if (!("storages" in this)) {
22624
21662
  this.storages = {};
22625
21663
  }
@@ -22637,6 +21675,7 @@ const BankOfCanada = (function() {
22637
21675
  this.source.fieldsSchema[nodeName].fields,
22638
21676
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
22639
21677
  );
21678
+ await this.storages[nodeName].init();
22640
21679
  }
22641
21680
  return this.storages[nodeName];
22642
21681
  }
@@ -22652,8 +21691,8 @@ const BankOfCanada = (function() {
22652
21691
  const endDate = new Date(startDate);
22653
21692
  endDate.setDate(endDate.getDate() + daysToFetch - 1);
22654
21693
  return {
22655
- startDate: EnvironmentAdapter3.formatDate(startDate, "UTC", "yyyy-MM-dd"),
22656
- endDate: EnvironmentAdapter3.formatDate(endDate, "UTC", "yyyy-MM-dd")
21694
+ startDate: DateUtils3.formatDate(startDate),
21695
+ endDate: DateUtils3.formatDate(endDate)
22657
21696
  };
22658
21697
  }
22659
21698
  };
@@ -22699,7 +21738,6 @@ const AvailableConnectors = [
22699
21738
  "BankOfCanada"
22700
21739
  ];
22701
21740
  const AvailableStorages = [
22702
- "GoogleSheets",
22703
21741
  "GoogleBigQuery",
22704
21742
  "AwsAthena"
22705
21743
  ];
@@ -22724,7 +21762,6 @@ const OWOX = {
22724
21762
  CriteoAds,
22725
21763
  BankOfCanada,
22726
21764
  // Individual storages
22727
- GoogleSheets,
22728
21765
  GoogleBigQuery,
22729
21766
  AwsAthena
22730
21767
  };