@owox/connectors 0.12.0-next-20251105102829 → 0.12.0-next-20251105123137

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -25,11 +25,6 @@ var require_index = __commonJS({
25
25
  SERVER_ERROR_MIN: 500,
26
26
  SERVER_ERROR_MAX: 599
27
27
  };
28
- var ENVIRONMENT = {
29
- UNKNOWN: 1,
30
- APPS_SCRIPT: 2,
31
- NODE: 3
32
- };
33
28
  var EXECUTION_STATUS = {
34
29
  IMPORT_IN_PROGRESS: 1,
35
30
  CLEANUP_IN_PROGRESS: 2,
@@ -77,269 +72,11 @@ var require_index = __commonJS({
77
72
  }
78
73
  class UnsupportedEnvironmentException extends AbstractException {
79
74
  }
80
- var EnvironmentAdapter = class EnvironmentAdapter {
81
- /**
82
- * Mac algorithm constants.
83
- *
84
- * @type {Object}
85
- */
86
- static get MacAlgorithm() {
87
- return {
88
- HMAC_SHA_256: "HMAC_SHA_256",
89
- HMAC_SHA_384: "HMAC_SHA_384",
90
- HMAC_SHA_512: "HMAC_SHA_512",
91
- HMAC_SHA_1: "HMAC_SHA_1",
92
- HMAC_MD5: "HMAC_MD5"
93
- };
94
- }
95
- constructor() {
96
- this.environment = this.getEnvironment();
97
- }
98
- /**
99
- * Get the current environment.
100
- * Detects whether code is running in Google Apps Script or Node.js environment.
101
- *
102
- * @returns {ENVIRONMENT} The detected environment (APPS_SCRIPT, NODE, or UNKNOWN)
103
- * @throws {UnsupportedEnvironmentException} If environment cannot be determined
104
- */
105
- static getEnvironment() {
106
- if (typeof this.environment !== "undefined") {
107
- return this.environment;
108
- }
109
- if (typeof UrlFetchApp !== "undefined") {
110
- this.environment = ENVIRONMENT.APPS_SCRIPT;
111
- } else if (typeof process !== "undefined") {
112
- this.environment = ENVIRONMENT.NODE;
113
- } else {
114
- this.environment = ENVIRONMENT.UNKNOWN;
115
- }
116
- return this.environment;
117
- }
118
- /**
119
- * Fetch data from the given URL.
120
- *
121
- * @param {string} url - The URL to fetch data from.
122
- * @param {Object} options - Options for the fetch request.
123
- * @returns {FetchResponse}
124
- *
125
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
126
- */
127
- static fetch(url, options = {}) {
128
- const env = this.getEnvironment();
129
- if (env === ENVIRONMENT.APPS_SCRIPT) {
130
- const response = UrlFetchApp.fetch(url, options);
131
- return this._wrapAppsScriptResponse(response);
132
- }
133
- if (env === ENVIRONMENT.NODE) {
134
- const method = options.method || "GET";
135
- const response = request(method, url, options);
136
- return this._wrapNodeResponse(response);
137
- }
138
- throw new UnsupportedEnvironmentException("Unsupported environment");
139
- }
140
- /**
141
- * Sleep for the given number of milliseconds.
142
- *
143
- * @param {number} ms - The number of milliseconds to sleep.
144
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
145
- */
146
- static sleep(ms) {
147
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
148
- Utilities.sleep(ms);
149
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
150
- let done = false;
151
- new Promise((resolve) => {
152
- setTimeout(() => {
153
- done = true;
154
- resolve();
155
- }, ms);
156
- });
157
- deasync.loopWhile(() => !done);
158
- } else {
159
- throw new UnsupportedEnvironmentException("Unsupported environment");
160
- }
161
- }
162
- /**
163
- * Format the given date.
164
- *
165
- * @param {Date} date - The date to format.
166
- * @param {string} timezone - The timezone to format the date in.
167
- * @param {string} format - The format to format the date in.
168
- * @returns {string}
169
- *
170
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
171
- */
172
- static formatDate(date, timezone, format) {
173
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
174
- return Utilities.formatDate(date, timezone, format);
175
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
176
- return date.toISOString().split("T")[0];
177
- } else {
178
- throw new UnsupportedEnvironmentException("Unsupported environment");
179
- }
180
- }
181
- /**
182
- * Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
183
- *
184
- * @returns {string} UUID
185
- *
186
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
187
- */
188
- static getUuid() {
189
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
190
- return Utilities.getUuid();
191
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
192
- const crypto = require("node:crypto");
193
- return crypto.randomUUID();
194
- } else {
195
- throw new UnsupportedEnvironmentException("Unsupported environment");
196
- }
197
- }
198
- /**
199
- * Encode the given data to base64.
200
- *
201
- * @param {string} data - The data to encode.
202
- * @returns {string}
203
- *
204
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
205
- */
206
- static base64Encode(data) {
207
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
208
- return Utilities.base64Encode(data);
209
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
210
- return Buffer.from(data).toString("base64");
211
- } else {
212
- throw new UnsupportedEnvironmentException("Unsupported environment");
213
- }
214
- }
215
- /**
216
- * Compute the HMAC signature for the given data.
217
- *
218
- * @param {string} algorithm - The algorithm to use.
219
- * @param {string} data - The data to compute the signature for.
220
- * @param {string} key - The key to use.
221
- * @returns {string}
222
- *
223
- * @throws {UnsupportedEnvironmentException} If the environment is not supported.
224
- */
225
- static computeHmacSignature(algorithm, data, key) {
226
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
227
- if (typeof algorithm === "string") {
228
- algorithm = Utilities.MacAlgorithm[algorithm];
229
- }
230
- return Utilities.computeHmacSignature(algorithm, data, key);
231
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
232
- const crypto = require("node:crypto");
233
- const algorithmMap = {
234
- "HMAC_SHA_256": "sha256",
235
- "HMAC_SHA_384": "sha384",
236
- "HMAC_SHA_512": "sha512",
237
- "HMAC_SHA_1": "sha1",
238
- "HMAC_MD5": "md5"
239
- };
240
- const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
241
- const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
242
- return Array.from(buffer);
243
- } else {
244
- throw new UnsupportedEnvironmentException("Unsupported environment");
245
- }
246
- }
247
- /**
248
- * Parse CSV string into array of arrays
249
- *
250
- * @param {string} csvString - The CSV string to parse
251
- * @param {string} [delimiter=','] - The delimiter to use for parsing CSV
252
- * @returns {Array<Array<string>>} Parsed CSV data
253
- * @throws {UnsupportedEnvironmentException} If the environment is not supported
254
- */
255
- static parseCsv(csvString, delimiter = ",") {
256
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
257
- return Utilities.parseCsv(csvString, delimiter);
258
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
259
- return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
260
- const trimmed = cell.trim();
261
- if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
262
- return trimmed.slice(1, -1).replace(/""/g, '"');
263
- }
264
- return trimmed;
265
- }));
266
- } else {
267
- throw new UnsupportedEnvironmentException("Unsupported environment");
268
- }
269
- }
270
- /**
271
- * Unzip a blob/buffer
272
- *
273
- * @param {Blob|Buffer} data - The data to unzip
274
- * @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
275
- * @throws {UnsupportedEnvironmentException} If the environment is not supported
276
- */
277
- static unzip(data) {
278
- if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
279
- return Utilities.unzip(data);
280
- } else if (this.getEnvironment() === ENVIRONMENT.NODE) {
281
- const zip = new AdmZip(data);
282
- return zip.getEntries().map((entry) => ({
283
- getDataAsString: () => entry.getData().toString("utf8")
284
- }));
285
- } else {
286
- throw new UnsupportedEnvironmentException("Unsupported environment");
287
- }
288
- }
289
- /**
290
- * Wraps the response from the Apps Script environment.
291
- * Not use directly, only for internal purposes.
292
- *
293
- * @param {Object} response
294
- * @returns {FetchResponse}
295
- */
296
- static _wrapAppsScriptResponse(response) {
297
- return {
298
- getHeaders: () => response.getAllHeaders(),
299
- getAsJson: () => {
300
- try {
301
- return JSON.parse(response.getContentText());
302
- } catch (e) {
303
- throw new Error("Invalid JSON response");
304
- }
305
- },
306
- getContent: () => response.getContent(),
307
- getContentText: () => response.getContentText(),
308
- getBlob: () => response.getBlob(),
309
- getResponseCode: () => response.getResponseCode()
310
- };
311
- }
312
- /**
313
- * Wraps the response from the Node environment.
314
- * Not use directly, only for internal purposes.
315
- *
316
- * @param {Object} response
317
- * @returns {FetchResponse}
318
- */
319
- static _wrapNodeResponse(response) {
320
- const headers = response.headers || {};
321
- const text = response.body ? response.body.toString() : "";
322
- return {
323
- getHeaders: () => headers,
324
- getAsJson: () => {
325
- try {
326
- return JSON.parse(text);
327
- } catch (e) {
328
- throw new Error("Invalid JSON response");
329
- }
330
- },
331
- getContent: () => text,
332
- getContentText: () => text,
333
- getBlob: () => response.body,
334
- getResponseCode: () => response.statusCode
335
- };
336
- }
337
- };
338
75
  class AbstractStorage {
339
76
  //---- constructor -------------------------------------------------
340
77
  /**
341
- * Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
342
- * @param config (object) instance of Sheet
78
+ * Abstract class for storage operations providing common methods for data persistence
79
+ * @param config (object) instance of AbstractConfig
343
80
  * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
344
81
  * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
345
82
  * @param description (string) string with storage description }
@@ -363,6 +100,14 @@ var require_index = __commonJS({
363
100
  }
364
101
  }
365
102
  //----------------------------------------------------------------
103
+ //---- init --------------------------------------------------------
104
+ /**
105
+ * Initializing storage
106
+ */
107
+ async init() {
108
+ throw new Error("Method init() has to be implemented in a child class of AbstractStorage");
109
+ }
110
+ //----------------------------------------------------------------
366
111
  //---- getUniqueKeyByRecordFields ----------------------------------
367
112
  /**
368
113
  * Calculcating unique key based on this.uniqueKeyColumns
@@ -419,11 +164,12 @@ var require_index = __commonJS({
419
164
  //----------------------------------------------------------------
420
165
  //---- saveData ----------------------------------------------------
421
166
  /**
422
- * Saving data to a storage. Has to be implemented in
167
+ * Saving data to a storage. Has to be implemented in child class as async method.
423
168
  * @param {data} array of assoc objects with records to save
169
+ * @returns {Promise<void>}
424
170
  */
425
- saveData(data) {
426
- throw new Error("Method saveDate() has to be implemented in a child class of AbstractStorage");
171
+ async saveData(data) {
172
+ throw new Error("Method saveData() has to be implemented in a child class of AbstractStorage");
427
173
  }
428
174
  //----------------------------------------------------------------
429
175
  //---- saveRecordsAddedToBuffer ------------------------------------
@@ -494,16 +240,6 @@ var require_index = __commonJS({
494
240
  return record;
495
241
  }
496
242
  //----------------------------------------------------------------
497
- //---- areHeadersNeeded --------------------------------------------
498
- /**
499
- * Checks if storage needs headers to be added
500
- * By default returns false, should be overridden in child classes if needed
501
- * @returns {boolean} true if headers need to be added, false otherwise
502
- */
503
- areHeadersNeeded() {
504
- return false;
505
- }
506
- //----------------------------------------------------------------
507
243
  //---- getSelectedFields -------------------------------------------
508
244
  /**
509
245
  * Parse Fields config value and return array of selected field names
@@ -557,7 +293,7 @@ var require_index = __commonJS({
557
293
  * A Data Source-specific methid is used to fetch new data and return it as an array of objects, where each property of an object corresponds to a column name.
558
294
  * @return data array
559
295
  */
560
- fetchData() {
296
+ async fetchData() {
561
297
  throw new Error("Method fetchData must be implemented in Class inheritor of AbstractSource");
562
298
  }
563
299
  //----------------------------------------------------------------
@@ -582,16 +318,16 @@ var require_index = __commonJS({
582
318
  * @return {HTTPResponse} The response object from the fetch
583
319
  * @throws {HttpRequestException} After exhausting all retries
584
320
  */
585
- urlFetchWithRetry(url, options) {
321
+ async urlFetchWithRetry(url, options) {
586
322
  for (let attempt = 1; attempt <= this.config.MaxFetchRetries.value; attempt++) {
587
323
  try {
588
- const response = EnvironmentAdapter.fetch(url, { ...options, muteHttpExceptions: true });
589
- return this._validateResponse(response);
324
+ const response = await HttpUtils.fetch(url, { ...options, muteHttpExceptions: true });
325
+ return await this._validateResponse(response);
590
326
  } catch (error) {
591
327
  if (!this._shouldRetry(error, attempt)) {
592
328
  throw error;
593
329
  }
594
- this._waitBeforeRetry(attempt);
330
+ await this._waitBeforeRetry(attempt);
595
331
  }
596
332
  }
597
333
  }
@@ -603,12 +339,12 @@ var require_index = __commonJS({
603
339
  * @throws {HttpRequestException} If the response indicates an error
604
340
  * @private
605
341
  */
606
- _validateResponse(response) {
342
+ async _validateResponse(response) {
607
343
  const code = response.getResponseCode();
608
344
  if (code >= HTTP_STATUS.SUCCESS_MIN && code <= HTTP_STATUS.SUCCESS_MAX) {
609
345
  return response;
610
346
  }
611
- const errorInfo = this._extractErrorInfo(response);
347
+ const errorInfo = await this._extractErrorInfo(response);
612
348
  throw new HttpRequestException({
613
349
  message: errorInfo.message,
614
350
  statusCode: code,
@@ -622,9 +358,9 @@ var require_index = __commonJS({
622
358
  * @return {Object} Object containing error message and JSON data if available
623
359
  * @private
624
360
  */
625
- _extractErrorInfo(response) {
361
+ async _extractErrorInfo(response) {
626
362
  var _a, _b;
627
- const text = response.getContentText();
363
+ const text = await response.getContentText();
628
364
  let parsedJson = null;
629
365
  let message = text;
630
366
  try {
@@ -661,10 +397,10 @@ var require_index = __commonJS({
661
397
  * @param {number} attempt - The current attempt number
662
398
  * @private
663
399
  */
664
- _waitBeforeRetry(attempt) {
400
+ async _waitBeforeRetry(attempt) {
665
401
  const delay = this.calculateBackoff(attempt);
666
402
  console.log(`Retrying after ${Math.round(delay / 1e3)}s...`);
667
- EnvironmentAdapter.sleep(delay);
403
+ await AsyncUtils.delay(delay);
668
404
  }
669
405
  //---- calculateBackoff --------------------------------------------
670
406
  /**
@@ -796,7 +532,7 @@ var require_index = __commonJS({
796
532
  /**
797
533
  * Initiates imports new data from a data source
798
534
  */
799
- run() {
535
+ async run() {
800
536
  try {
801
537
  if (this.config.isInProgress()) {
802
538
  this.config.logMessage("Import is already in progress");
@@ -806,11 +542,7 @@ var require_index = __commonJS({
806
542
  this.config.handleStatusUpdate({ status: EXECUTION_STATUS.IMPORT_IN_PROGRESS });
807
543
  this.config.updateLastImportDate();
808
544
  this.config.logMessage("Start importing new data");
809
- if (this.storage !== null && this.storage.areHeadersNeeded()) {
810
- this.storage.addHeader(this.storage.uniqueKeyColumns);
811
- this.config.logMessage(`Column(s) for unique key was added: ${this.storage.uniqueKeyColumns}`);
812
- }
813
- this.startImportProcess();
545
+ await this.startImportProcess();
814
546
  this.config.logMessage("Import is finished");
815
547
  this.config.handleStatusUpdate({
816
548
  status: EXECUTION_STATUS.IMPORT_DONE
@@ -831,7 +563,7 @@ var require_index = __commonJS({
831
563
  /**
832
564
  * A method for calling from Root script for determining parameters needed to fetch new data.
833
565
  */
834
- startImportProcess() {
566
+ async startImportProcess() {
835
567
  var _a;
836
568
  let startDate = null;
837
569
  let endDate = /* @__PURE__ */ new Date();
@@ -842,10 +574,10 @@ var require_index = __commonJS({
842
574
  return;
843
575
  }
844
576
  endDate.setDate(startDate.getDate() + daysToFetch);
845
- let data = this.source.fetchData(startDate, endDate);
577
+ let data = await this.source.fetchData(startDate, endDate);
846
578
  this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
847
579
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value) === "true") {
848
- this.storage.saveData(data);
580
+ await this.storage.saveData(data);
849
581
  }
850
582
  if (this.runConfig.type === RUN_CONFIG_TYPE.INCREMENTAL) {
851
583
  this.config.updateLastRequstedDate(endDate);
@@ -1002,31 +734,12 @@ var require_index = __commonJS({
1002
734
  * @param (object) with config data. Properties are parameters names, values are values
1003
735
  */
1004
736
  constructor(configData) {
1005
- this.addParameter("Environment", {
1006
- value: AbstractConfig.detectEnvironment(),
1007
- requiredType: "number",
1008
- attributes: [CONFIG_ATTRIBUTES.HIDE_IN_CONFIG_FORM, CONFIG_ATTRIBUTES.ADVANCED]
1009
- });
1010
737
  for (var name in configData) {
1011
738
  this.addParameter(name, configData[name]);
1012
739
  }
1013
740
  return this;
1014
741
  }
1015
742
  //----------------------------------------------------------------
1016
- //---- static helper -------------------------------------------------
1017
- /**
1018
- * Determines the runtime environment
1019
- * @returns {ENVIRONMENT} The detected environment
1020
- */
1021
- static detectEnvironment() {
1022
- if (typeof UrlFetchApp !== "undefined") {
1023
- return ENVIRONMENT.APPS_SCRIPT;
1024
- }
1025
- if (typeof process !== "undefined") {
1026
- return ENVIRONMENT.NODE;
1027
- }
1028
- return ENVIRONMENT.UNKNOWN;
1029
- }
1030
743
  //---- mergeParameters ---------------------------------------------
1031
744
  /**
1032
745
  * Merge configuration to existing config
@@ -1233,11 +946,11 @@ var require_index = __commonJS({
1233
946
  }
1234
947
  //----------------------------------------------------------------
1235
948
  }
1236
- function processShortLinks(data, { shortLinkField, urlFieldName }) {
949
+ async function processShortLinks(data, { shortLinkField, urlFieldName }) {
1237
950
  if (!Array.isArray(data) || data.length === 0) return data;
1238
951
  const shortLinks = _collectUniqueShortLinks(data, shortLinkField, urlFieldName);
1239
952
  if (shortLinks.length === 0) return data;
1240
- const resolvedShortLinks = _resolveShortLinks(shortLinks);
953
+ const resolvedShortLinks = await _resolveShortLinks(shortLinks);
1241
954
  return _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName);
1242
955
  }
1243
956
  function _collectUniqueShortLinks(data, shortLinkField, urlFieldName) {
@@ -1259,13 +972,11 @@ var require_index = __commonJS({
1259
972
  if (hasParams) return false;
1260
973
  return /^https:\/\/[^\/]+\/[^\/]+$/.test(url);
1261
974
  }
1262
- function _resolveShortLinks(shortLinks) {
1263
- return shortLinks.map((linkObj) => {
975
+ async function _resolveShortLinks(shortLinks) {
976
+ const promises = shortLinks.map(async (linkObj) => {
1264
977
  try {
1265
- const response = EnvironmentAdapter.fetch(linkObj.originalUrl, {
1266
- method: "GET",
1267
- followRedirects: false,
1268
- muteHttpExceptions: true
978
+ const response = await HttpUtils.fetch(linkObj.originalUrl, {
979
+ method: "GET"
1269
980
  });
1270
981
  const headers = response.getHeaders();
1271
982
  const resolvedUrl = headers.Location || headers.location || linkObj.originalUrl;
@@ -1281,6 +992,7 @@ var require_index = __commonJS({
1281
992
  };
1282
993
  }
1283
994
  });
995
+ return Promise.all(promises);
1284
996
  }
1285
997
  function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName) {
1286
998
  return data.map((record) => {
@@ -1304,15 +1016,15 @@ var require_index = __commonJS({
1304
1016
  var OAuthUtils = {
1305
1017
  /**
1306
1018
  * Universal OAuth access token retrieval method
1307
- *
1019
+ *
1308
1020
  * @param {Object} options - All configuration options
1309
1021
  * @param {Object} options.config - Configuration object containing credentials
1310
1022
  * @param {string} options.tokenUrl - OAuth token endpoint URL
1311
1023
  * @param {Object} options.formData - Form data to send in request body
1312
1024
  * @param {Object} [options.headers] - Request headers
1313
- * @returns {string} - The access token
1025
+ * @returns {Promise<string>} - The access token
1314
1026
  */
1315
- getAccessToken({ config, tokenUrl, formData, headers = {} }) {
1027
+ async getAccessToken({ config, tokenUrl, formData, headers = {} }) {
1316
1028
  const requestHeaders = {
1317
1029
  "Content-Type": "application/x-www-form-urlencoded",
1318
1030
  ...headers
@@ -1325,8 +1037,9 @@ var require_index = __commonJS({
1325
1037
  body: Object.entries(formData).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
1326
1038
  };
1327
1039
  try {
1328
- const resp = EnvironmentAdapter.fetch(tokenUrl, options);
1329
- const json = JSON.parse(resp.getContentText());
1040
+ const resp = await HttpUtils.fetch(tokenUrl, options);
1041
+ const text = await resp.getContentText();
1042
+ const json = JSON.parse(text);
1330
1043
  if (json.error) {
1331
1044
  throw new Error(`Token error: ${json.error}`);
1332
1045
  }
@@ -1339,7 +1052,7 @@ var require_index = __commonJS({
1339
1052
  },
1340
1053
  /**
1341
1054
  * Get access token using Service Account JWT authentication
1342
- *
1055
+ *
1343
1056
  * @param {Object} options - Configuration options
1344
1057
  * @param {Object} options.config - Configuration object
1345
1058
  * @param {string} options.tokenUrl - Token URL
@@ -1347,7 +1060,7 @@ var require_index = __commonJS({
1347
1060
  * @param {string} options.scope - OAuth scope (e.g., "https://www.googleapis.com/auth/adwords")
1348
1061
  * @returns {string} - The access token
1349
1062
  */
1350
- getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
1063
+ async getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
1351
1064
  try {
1352
1065
  const serviceAccountData = JSON.parse(serviceAccountKeyJson);
1353
1066
  const now = Math.floor(Date.now() / 1e3);
@@ -1365,7 +1078,7 @@ var require_index = __commonJS({
1365
1078
  grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
1366
1079
  assertion: jwt
1367
1080
  };
1368
- const accessToken = this.getAccessToken({
1081
+ const accessToken = await this.getAccessToken({
1369
1082
  config,
1370
1083
  tokenUrl,
1371
1084
  formData
@@ -1418,6 +1131,68 @@ var require_index = __commonJS({
1418
1131
  return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
1419
1132
  }
1420
1133
  };
1134
+ var HttpUtils = class HttpUtils {
1135
+ /**
1136
+ * Fetch data from the given URL.
1137
+ *
1138
+ * @param {string} url - The URL to fetch data from.
1139
+ * @param {Object} options - Options for the fetch request.
1140
+ * @returns {Promise<FetchResponse>}
1141
+ */
1142
+ static async fetch(url, options = {}) {
1143
+ const method = options.method || "GET";
1144
+ const fetchOptions = {
1145
+ method: method.toUpperCase(),
1146
+ headers: options.headers || {}
1147
+ };
1148
+ if (options.body) {
1149
+ fetchOptions.body = options.body;
1150
+ }
1151
+ const response = await fetch(url, fetchOptions);
1152
+ return this._wrapNodeResponse(response);
1153
+ }
1154
+ /**
1155
+ * Wraps the response from the Node environment.
1156
+ * Not use directly, only for internal purposes.
1157
+ *
1158
+ * @param {Response} response - Native fetch Response object
1159
+ * @returns {FetchResponse}
1160
+ */
1161
+ static _wrapNodeResponse(response) {
1162
+ let textCache = null;
1163
+ let blobCache = null;
1164
+ const getText = async () => {
1165
+ if (textCache === null) {
1166
+ textCache = await response.text();
1167
+ }
1168
+ return textCache;
1169
+ };
1170
+ const getBlob = async () => {
1171
+ if (blobCache === null) {
1172
+ blobCache = await response.arrayBuffer();
1173
+ }
1174
+ return Buffer.from(blobCache);
1175
+ };
1176
+ const headersObj = {};
1177
+ response.headers.forEach((value, key) => {
1178
+ headersObj[key] = value;
1179
+ });
1180
+ return {
1181
+ getHeaders: () => headersObj,
1182
+ getAsJson: () => getText().then((text) => {
1183
+ try {
1184
+ return JSON.parse(text);
1185
+ } catch (e) {
1186
+ throw new Error("Invalid JSON response");
1187
+ }
1188
+ }),
1189
+ getContent: () => getText(),
1190
+ getContentText: () => getText(),
1191
+ getBlob: () => getBlob(),
1192
+ getResponseCode: () => response.status
1193
+ };
1194
+ }
1195
+ };
1421
1196
  var FormatUtils = {
1422
1197
  /**
1423
1198
  * Universal ID parser: parses comma/semicolon separated string to array of numeric IDs
@@ -1460,6 +1235,102 @@ var require_index = __commonJS({
1460
1235
  }, {});
1461
1236
  }
1462
1237
  };
1238
+ var FileUtils = class FileUtils {
1239
+ /**
1240
+ * Parse CSV string into array of arrays
1241
+ *
1242
+ * @param {string} csvString - The CSV string to parse
1243
+ * @param {string} [delimiter=','] - The delimiter to use for parsing CSV
1244
+ * @returns {Array<Array<string>>} Parsed CSV data
1245
+ */
1246
+ static parseCsv(csvString, delimiter = ",") {
1247
+ return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
1248
+ const trimmed = cell.trim();
1249
+ if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
1250
+ return trimmed.slice(1, -1).replace(/""/g, '"');
1251
+ }
1252
+ return trimmed;
1253
+ }));
1254
+ }
1255
+ /**
1256
+ * Unzip a blob/buffer
1257
+ *
1258
+ * @param {Buffer} data - The data to unzip
1259
+ * @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
1260
+ */
1261
+ static unzip(data) {
1262
+ const zip = new AdmZip(data);
1263
+ return zip.getEntries().map((entry) => ({
1264
+ getDataAsString: () => entry.getData().toString("utf8")
1265
+ }));
1266
+ }
1267
+ };
1268
+ var DateUtils = class DateUtils {
1269
+ /**
1270
+ * Format the given date to ISO format (YYYY-MM-DD).
1271
+ *
1272
+ * @param {Date} date - The date to format.
1273
+ * @returns {string} ISO formatted date (YYYY-MM-DD)
1274
+ */
1275
+ static formatDate(date) {
1276
+ return date.toISOString().split("T")[0];
1277
+ }
1278
+ };
1279
+ var CryptoUtils = class CryptoUtils {
1280
+ /**
1281
+ * Mac algorithm constants.
1282
+ *
1283
+ * @type {Object}
1284
+ */
1285
+ static get MacAlgorithm() {
1286
+ return {
1287
+ HMAC_SHA_256: "HMAC_SHA_256",
1288
+ HMAC_SHA_384: "HMAC_SHA_384",
1289
+ HMAC_SHA_512: "HMAC_SHA_512",
1290
+ HMAC_SHA_1: "HMAC_SHA_1",
1291
+ HMAC_MD5: "HMAC_MD5"
1292
+ };
1293
+ }
1294
+ /**
1295
+ * Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
1296
+ *
1297
+ * @returns {string} UUID
1298
+ */
1299
+ static getUuid() {
1300
+ const crypto = require("node:crypto");
1301
+ return crypto.randomUUID();
1302
+ }
1303
+ /**
1304
+ * Encode the given data to base64.
1305
+ *
1306
+ * @param {string} data - The data to encode.
1307
+ * @returns {string}
1308
+ */
1309
+ static base64Encode(data) {
1310
+ return Buffer.from(data).toString("base64");
1311
+ }
1312
+ /**
1313
+ * Compute the HMAC signature for the given data.
1314
+ *
1315
+ * @param {string} algorithm - The algorithm to use (e.g., 'HMAC_SHA_256', 'sha256').
1316
+ * @param {string} data - The data to compute the signature for.
1317
+ * @param {string} key - The key to use.
1318
+ * @returns {Array<number>} Byte array of the signature
1319
+ */
1320
+ static computeHmacSignature(algorithm, data, key) {
1321
+ const crypto = require("node:crypto");
1322
+ const algorithmMap = {
1323
+ "HMAC_SHA_256": "sha256",
1324
+ "HMAC_SHA_384": "sha384",
1325
+ "HMAC_SHA_512": "sha512",
1326
+ "HMAC_SHA_1": "sha1",
1327
+ "HMAC_MD5": "md5"
1328
+ };
1329
+ const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
1330
+ const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
1331
+ return Array.from(buffer);
1332
+ }
1333
+ };
1463
1334
  var ConnectorUtils = {
1464
1335
  /**
1465
1336
  * Check if a node is a time series node
@@ -1482,6 +1353,17 @@ var require_index = __commonJS({
1482
1353
  }, {});
1483
1354
  }
1484
1355
  };
1356
+ var AsyncUtils = class AsyncUtils {
1357
+ /**
1358
+ * Async delay for the given number of milliseconds.
1359
+ *
1360
+ * @param {number} ms - The number of milliseconds to delay.
1361
+ * @returns {Promise<void>}
1362
+ */
1363
+ static async delay(ms) {
1364
+ return new Promise((resolve) => setTimeout(resolve, ms));
1365
+ }
1366
+ };
1485
1367
  class RunConfigDto {
1486
1368
  constructor(config) {
1487
1369
  this._config = config;
@@ -1893,902 +1775,38 @@ var require_index = __commonJS({
1893
1775
  );
1894
1776
  }
1895
1777
  }
1896
- var GoogleSheetsConfig = class GoogleSheetsConfig extends AbstractConfig {
1897
- //---- constructor -------------------------------------------------
1898
- constructor(configRange) {
1899
- if (typeof configRange.getA1Notation !== "function") {
1900
- throw new Error(`Unable to create an GoogleSheetsConfig object. The first constructor's parameter must be an SpreadsheetApp.Sheet object`);
1901
- }
1902
- let configObject = {};
1903
- configRange.getValues().forEach((row, index) => {
1904
- var name = row[0].replaceAll(/[^a-zA-Z0-9]/gi, "");
1905
- if (!["", "Parameters", "Status", "Name"].includes(name)) {
1906
- var value = row[1];
1907
- configObject[name] = {};
1908
- if (value || value === 0) {
1909
- configObject[name].value = value;
1910
- }
1911
- if (row[0].slice(-1) == "*") {
1912
- configObject[name].isRequired = true;
1913
- }
1914
- if (["Log", "CurrentStatus", "LastImportDate", "LastRequestedDate", "DestinationSpreadsheet"].includes(name)) {
1915
- configObject[name].cell = configRange.offset(index, 1, 1, 1);
1916
- }
1917
- }
1918
- });
1919
- configObject.configSpreadsheet = configRange.getSheet().getParent();
1920
- configObject.Log.timeZone = configRange.getSheet().getParent().getSpreadsheetTimeZone();
1921
- super(configObject);
1922
- this.mergeParameters({
1923
- MaxRunTimeout: {
1924
- isRequired: true,
1925
- requiredType: "number",
1926
- default: 30
1927
- },
1928
- NotifyByEmail: {
1929
- isRequired: false,
1930
- requiredType: "string",
1931
- default: ""
1932
- },
1933
- NotifyByGoogleChat: {
1934
- isRequired: false,
1935
- requiredType: "string",
1936
- default: ""
1937
- },
1938
- NotifyWhen: {
1939
- isRequired: false,
1940
- requiredType: "string",
1941
- default: "Never"
1942
- }
1943
- });
1944
- }
1945
- //---- handleStatusUpdate -----------------------------------------------
1946
- /**
1947
- * @param {Object} params - Parameters object with status and other properties
1948
- * @param {number} params.status - Status constant
1949
- * @param {string} params.error - Error message for Error status
1950
- */
1951
- handleStatusUpdate({ status, error }) {
1952
- this.manageTimeoutTrigger(status);
1953
- this.updateCurrentStatus(status);
1954
- if (this.shouldSendNotifications(status)) {
1955
- this.sendNotifications({ status, error });
1956
- }
1957
- }
1958
- //----------------------------------------------------------------
1959
- //---- manageTimeoutTrigger ----------------------------------------
1960
- /**
1961
- * Manage timeout trigger based on current status
1962
- * @param {number} status - Status constant
1963
- */
1964
- manageTimeoutTrigger(status) {
1965
- if (status === EXECUTION_STATUS.IMPORT_IN_PROGRESS) {
1966
- this.createTimeoutTrigger();
1967
- } else if (status === EXECUTION_STATUS.IMPORT_DONE || status === EXECUTION_STATUS.ERROR) {
1968
- this.removeTimeoutTrigger();
1969
- }
1970
- }
1971
- //---- getStatusProperties ------------------------------------------
1972
- /**
1973
- * Get all properties for a given status
1974
- * @param {number} status - Status constant
1975
- * @returns {Object} - Object with all status properties
1976
- */
1977
- getStatusProperties(status) {
1978
- switch (status) {
1979
- case EXECUTION_STATUS.IMPORT_IN_PROGRESS:
1980
- return {
1981
- displayText: "Import in progress",
1982
- backgroundColor: "#c9e3f9",
1983
- notificationMessage: "Import is in progress."
1984
- };
1985
- case EXECUTION_STATUS.CLEANUP_IN_PROGRESS:
1986
- return {
1987
- displayText: "CleanUp in progress",
1988
- backgroundColor: "#c9e3f9",
1989
- notificationMessage: "Cleanup is in progress."
1990
- };
1991
- case EXECUTION_STATUS.IMPORT_DONE:
1992
- return {
1993
- displayText: "Done",
1994
- backgroundColor: "#d4efd5",
1995
- notificationMessage: "Import completed successfully."
1996
- };
1997
- case EXECUTION_STATUS.CLEANUP_DONE:
1998
- return {
1999
- displayText: "Done",
2000
- backgroundColor: "#d4efd5",
2001
- notificationMessage: "Cleanup completed successfully."
2002
- };
2003
- case EXECUTION_STATUS.ERROR:
2004
- return {
2005
- displayText: "Error",
2006
- backgroundColor: "#fdd2cf",
2007
- notificationMessage: "Error occurred"
2008
- };
2009
- default:
2010
- throw new Error(`Unknown status constant: ${status}`);
2011
- }
2012
- }
2013
- //----------------------------------------------------------------
2014
- //---- updateCurrentStatus -----------------------------------------
2015
- /**
2016
- * @param {number} status - Status constant
2017
- */
2018
- updateCurrentStatus(status) {
2019
- const statusProps = this.getStatusProperties(status);
2020
- this.CurrentStatus.cell.setValue(statusProps.displayText);
2021
- this.CurrentStatus.cell.setBackground(statusProps.backgroundColor);
2022
- }
2023
- //----------------------------------------------------------------
2024
- //---- updateLastImportDate ----------------------------------------
2025
- /**
2026
- * updating the last import attempt date in a config sheet
2027
- */
2028
- updateLastImportDate() {
2029
- this.LastImportDate.cell.setValue(
2030
- EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss")
2031
- );
2032
- }
2033
- //----------------------------------------------------------------
2034
- //---- updateLastRequstedDate --------------------------------------
2035
- /**
2036
- * Updating the last requested date in a config sheet
2037
- * @param date Date requested date
2038
- */
2039
- updateLastRequstedDate(date) {
2040
- if ("LastRequestedDate" in this && (!this.LastRequestedDate.value || date.getTime() != this.LastRequestedDate.value.getTime())) {
2041
- this.LastRequestedDate.value = new Date(date.getTime());
2042
- this.LastRequestedDate.cell.setValue(EnvironmentAdapter.formatDate(date, this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"));
2043
- }
2044
- }
2045
- //----------------------------------------------------------------
2046
- //---- updateFieldsSheet -------------------------------------------
2047
- /**
2048
- * Updating the content of the fields list to simplify the selection of fields for import
2049
- * @param connector AbstractSource
2050
- * @param sheetName string, Fields by default
2051
- */
2052
- updateFieldsSheet(source, sheetName = "Fields") {
2053
- this.validate();
2054
- var configSheetStorage = new GoogleSheetsStorage(
2055
- this.mergeParameters({
2056
- DestinationSheetName: {
2057
- // @TODO: make sure it would affect destination sheet for import data. CONFIG is an object. Probably we might dublicate it
2058
- value: sheetName
2059
- }
2060
- }),
2061
- ["id"]
2062
- );
2063
- var groups = source.getFieldsSchema();
2064
- var data = [];
2065
- for (var groupName in groups) {
2066
- data.push({
2067
- "id": groupName,
2068
- "✔️": groupName,
2069
- "name": "",
2070
- "description": `=IF( COUNTIFS(A:A, "${groupName} *", B:B, TRUE) > 0, COUNTIFS(A:A, "${groupName} *", B:B, TRUE), "")`
2071
- });
2072
- data.push({ "id": `${groupName} !desc`, "✔️": groups[groupName].description });
2073
- data.push({ "id": `${groupName} !doc`, "✔️": groups[groupName].documentation });
2074
- if (groups[groupName].fields) {
2075
- for (var fieldName in groups[groupName].fields) {
2076
- data.push({
2077
- "id": groupName + " " + fieldName,
2078
- "name": fieldName,
2079
- "description": groups[groupName].fields[fieldName].description
2080
- });
2081
- }
2082
- data.push({ "id": `${groupName} zzz_separator` });
2083
- }
2084
- }
2085
- for (var groupName in groups) {
2086
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2087
- if (groupRow !== null) {
2088
- let depth = configSheetStorage.SHEET.getRowGroupDepth(groupRow.rowIndex + 5);
2089
- if (depth > 0) {
2090
- configSheetStorage.SHEET.getRowGroup(groupRow.rowIndex + 5, depth).remove();
2091
- }
2092
- }
2093
- }
2094
- configSheetStorage.saveData(data);
2095
- configSheetStorage.SHEET.sort(configSheetStorage.getColumnIndexByName("id"));
2096
- configSheetStorage.loadDataFromSheet();
2097
- configSheetStorage.SHEET.hideColumns(configSheetStorage.getColumnIndexByName("id"));
2098
- configSheetStorage.SHEET.setColumnWidth(configSheetStorage.getColumnIndexByName("✔️"), 25);
2099
- configSheetStorage.SHEET.autoResizeColumn(3);
2100
- var checkboxRule = SpreadsheetApp.newDataValidation().requireCheckbox().build();
2101
- for (var groupName in groups) {
2102
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2103
- var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 2}`);
2104
- range.setFontWeight("bold").setBackground("black").setFontColor("white");
2105
- var groupRow = configSheetStorage.getRecordByRecordFields({ "id": `${groupName} !desc` });
2106
- var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 3}`);
2107
- range.setBackground("#f3f3f3");
2108
- var checkboxesColumnIndex = configSheetStorage.getColumnIndexByName("✔️");
2109
- var groupNameRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
2110
- var groupFieldsCount = this.getFieldsCountByGroupName(configSheetStorage, groupName);
2111
- configSheetStorage.SHEET.getRange(groupNameRow.rowIndex + 5, checkboxesColumnIndex, groupFieldsCount).setDataValidation(checkboxRule);
2112
- configSheetStorage.SHEET.getRange(`${groupNameRow.rowIndex + 5}:${groupNameRow.rowIndex + 4 + groupFieldsCount}`).shiftRowGroupDepth(1);
2113
- }
2114
- configSheetStorage.SHEET.collapseAllRowGroups();
2115
- }
2116
- //----------------------------------------------------------------
2117
- //---- getFieldsCountByGroupName -----------------------------------
2118
- /**
2119
- * @param GoogleSheetsConfig object
2120
- * @param groupName string name of the group
2121
- * @return integer number of fields of requested group
2122
- */
2123
- getFieldsCountByGroupName(configSheetStorage, groupName) {
2124
- return Object.values(configSheetStorage.values).filter((element) => element.name && element.id.startsWith(`${groupName} `)).length;
2125
- }
2126
- //----------------------------------------------------------------
2127
- //---- isInProgress ------------------------------------------------
2128
- /**
2129
- * Checking current status if it is in progress or not
2130
- * @return boolean true is process in progress
2131
- */
2132
- isInProgress() {
2133
- let isInProgress = null;
2134
- if (this.CurrentStatus.cell.getValue().indexOf("progress") !== -1) {
2135
- let diff = (/* @__PURE__ */ new Date() - new Date(EnvironmentAdapter.formatDate(this.LastImportDate.cell.getValue(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"))) / (1e3 * 60);
2136
- isInProgress = diff < this.MaxRunTimeout.value;
2137
- } else {
2138
- isInProgress = false;
2139
- }
2140
- return isInProgress;
2141
- }
2142
- //----------------------------------------------------------------
2143
- //---- addWarningToCurrentStatus -----------------------------------
2144
- addWarningToCurrentStatus() {
2145
- this.CurrentStatus.cell.setBackground("#fff0c4");
2146
- }
2147
- //----------------------------------------------------------------
2148
- //---- validate ----------------------------------------------------
2149
- /**
2150
- * validating if google sheets config is correct
2151
- */
2152
- validate() {
2153
- let scriptTimeZone = Session.getScriptTimeZone();
2154
- if (scriptTimeZone != "Etc/UTC") {
2155
- throw new Error(`The Apps Script time zone must be set to UTC to avoid confusion with dates. Currently, it is set to ${scriptTimeZone} instead. Update the time zone in Project Settings`);
2156
- }
2157
- super.validate();
2158
- }
2159
- //----------------------------------------------------------------
2160
- //---- logMessage --------------------------------------------------
2161
- /**
2162
- * @param string message to Log
2163
- */
2164
- logMessage(message, removeExistingMessage = false) {
2165
- console.log(message);
2166
- let formattedDate = EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss");
2167
- let currentLog = removeExistingMessage ? "" : this.Log.cell.getValue();
2168
- currentLog ? currentLog += "\n" : "";
2169
- let emoji = "☑️ ";
2170
- let match;
2171
- if (match = message.match(new RegExp("^(\\p{Emoji_Presentation}|\\p{Emoji}\\uFE0F|\\p{Extended_Pictographic})\\s+", "u"))) {
2172
- emoji = match[0];
2173
- message = message.slice(2).trim();
2174
- }
2175
- this.Log.cell.setValue(
2176
- `${currentLog}${emoji}${formattedDate}: ${message}`
2177
- );
2178
- this.updateLastImportDate();
2179
- }
2180
- showCredentialsDialog(source) {
2181
- const ui = SpreadsheetApp.getUi();
2182
- const template = HtmlService.createTemplateFromFile("Views/credentials-input-dialog");
2183
- template.source = source;
2184
- const html = template.evaluate().setWidth(400).setHeight(450);
2185
- ui.showModalDialog(html, `${source.constructor.name} Credentials`);
2186
- }
2187
- showManualBackfillDialog(source) {
2188
- const ui = SpreadsheetApp.getUi();
2189
- const template = HtmlService.createTemplateFromFile("Views/manual-backfill-dialog");
2190
- template.source = source;
2191
- const html = template.evaluate().setWidth(600).setHeight(300);
2192
- ui.showModalDialog(html, "Manual Backfill");
2193
- }
2194
- //---- sendNotifications -------------------------------------------
2195
- /**
2196
- * Send notifications based on configuration settings
2197
- * @param {Object} params - Parameters object
2198
- * @param {string} params.status - Current status value
2199
- * @param {string} params.error - Error message for Error status
2200
- */
2201
- sendNotifications({ status, error }) {
2202
- try {
2203
- const { title, messageWithDetails } = this.prepareNotificationContent({ status, error });
2204
- if (this.NotifyByEmail && this.NotifyByEmail.value && this.NotifyByEmail.value.trim()) {
2205
- EmailNotification.send({
2206
- to: this.NotifyByEmail.value,
2207
- subject: title,
2208
- message: messageWithDetails
2209
- });
2210
- }
2211
- if (this.NotifyByGoogleChat && this.NotifyByGoogleChat.value && this.NotifyByGoogleChat.value.trim()) {
2212
- GoogleChatNotification.send({
2213
- webhookUrl: this.NotifyByGoogleChat.value.trim(),
2214
- message: messageWithDetails
2215
- });
2216
- }
2217
- } catch (error2) {
2218
- this.logMessage(`⚠️ Notification error: ${error2.message}`);
2219
- }
2220
- }
2221
- //----------------------------------------------------------------
2222
- //---- prepareNotificationContent ----------------------------------
2223
- /**
2224
- * Prepare notification title and message content
2225
- * @param {Object} params - Parameters object
2226
- * @param {number} params.status - Status constant
2227
- * @param {string} params.error - Error message for Error status
2228
- * @returns {Object} - Object with title and messageWithDetails properties
2229
- */
2230
- prepareNotificationContent({ status, error }) {
2231
- const documentName = this.configSpreadsheet.getName();
2232
- const documentUrl = this.configSpreadsheet.getUrl();
2233
- const { displayText, notificationMessage } = this.getStatusProperties(status);
2234
- const title = `${documentName} - Status: ${displayText}`;
2235
- return {
2236
- title,
2237
- messageWithDetails: `${title}
2238
- ${documentUrl}
2239
-
2240
- ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}` : ""}`
2241
- };
2242
- }
2243
- //----------------------------------------------------------------
2244
- //---- shouldSendNotifications -------------------------------------
2245
- /**
2246
- * Determine if notifications should be sent based on status and filter setting
2247
- * @param {number} status - Status constant
2248
- * @returns {boolean} - True if notifications should be sent
2249
- */
2250
- shouldSendNotifications(status) {
2251
- var _a;
2252
- const notifyWhen = (_a = this.NotifyWhen) == null ? void 0 : _a.value;
2253
- switch (notifyWhen) {
2254
- case "On error":
2255
- return status === EXECUTION_STATUS.ERROR;
2256
- case "On success":
2257
- return status === EXECUTION_STATUS.IMPORT_DONE;
2258
- case "Always":
2259
- return status === EXECUTION_STATUS.ERROR || status === EXECUTION_STATUS.IMPORT_DONE;
2260
- case "Never":
2261
- case "":
2262
- default:
2263
- return false;
2264
- }
2265
- }
2266
- //----------------------------------------------------------------
2267
- //---- createTimeoutTrigger ----------------------------------------
2268
- createTimeoutTrigger() {
2269
- this.removeTimeoutTrigger();
2270
- ScriptApp.newTrigger("checkForTimeout").timeBased().after((this.MaxRunTimeout.value * 2 + 1) * 60 * 1e3).create();
2271
- }
2272
- //----------------------------------------------------------------
2273
- //---- removeTimeoutTrigger ----------------------------------------
2274
- removeTimeoutTrigger() {
2275
- const triggers = ScriptApp.getProjectTriggers();
2276
- let removedCount = 0;
2277
- triggers.forEach((trigger) => {
2278
- if (trigger.getHandlerFunction() === "checkForTimeout") {
2279
- ScriptApp.deleteTrigger(trigger);
2280
- removedCount++;
2281
- }
2282
- });
2283
- console.log(`[TimeoutTrigger] ${removedCount > 0 ? `Removed ${removedCount} timeout trigger(s)` : "No timeout triggers found to remove"}`);
2284
- }
2285
- //----------------------------------------------------------------
2286
- //---- checkForTimeout ---------------------------------------------
2287
- checkForTimeout() {
2288
- if (!this.isInProgress()) {
2289
- console.log("[TimeoutTrigger] Status is NOT in progress, setting to Error and sending notification");
2290
- this.handleStatusUpdate({
2291
- status: EXECUTION_STATUS.ERROR,
2292
- error: "Import was interrupted (likely due to timeout)"
2293
- });
2294
- } else {
2295
- console.log("[TimeoutTrigger] Status is still in progress");
2296
- }
2297
- }
2298
- //----------------------------------------------------------------
2299
- };
2300
- var GoogleChatNotification = class GoogleChatNotification {
2301
- /**
2302
- * Send Google Chat notification
2303
- * @param {Object} params - Parameters object
2304
- * @param {string} params.webhookUrl - Google Chat webhook URL
2305
- * @param {string} params.message - Formatted notification message
2306
- */
2307
- static send(params) {
2308
- const { webhookUrl, message } = params;
2309
- if (!webhookUrl || !webhookUrl.trim()) {
2310
- return;
2311
- }
2312
- try {
2313
- const response = EnvironmentAdapter.fetch(webhookUrl.trim(), {
2314
- method: "POST",
2315
- headers: {
2316
- "Content-Type": "application/json; charset=UTF-8"
2317
- },
2318
- payload: JSON.stringify({ text: message })
2319
- });
2320
- if (response.getResponseCode() === 200) {
2321
- console.log("Google Chat notification sent successfully");
2322
- } else {
2323
- console.error(`Google Chat notification failed with status: ${response.getResponseCode()}`);
2324
- }
2325
- } catch (error) {
2326
- console.error("Failed to send Google Chat notification:", error);
2327
- }
2328
- }
2329
- };
2330
- var EmailNotification = class EmailNotification {
2331
- /**
2332
- * Send email notification
2333
- * @param {Object} params - Parameters object
2334
- * @param {string} params.to - Email address(es) to send to (can be multiple separated by commas)
2335
- * @param {string} params.subject - Email subject line
2336
- * @param {string} params.message - Formatted notification message
2337
- */
2338
- static send(params) {
2339
- const { to, subject, message } = params;
2340
- if (!to || !to.trim()) {
2341
- return;
2342
- }
2343
- try {
2344
- const emailAddresses = to.split(",").map((email) => email.trim()).filter((email) => email.length > 0).filter((email) => this.isValidEmail(email)).join(",");
2345
- if (!emailAddresses) {
2346
- console.log("Email notification skipped: no valid email addresses found");
2347
- return;
2348
- }
2349
- MailApp.sendEmail(
2350
- emailAddresses,
2351
- subject,
2352
- message,
2353
- { noReply: true }
2354
- );
2355
- console.log(`Email notification sent successfully to: ${emailAddresses}`);
2356
- } catch (error) {
2357
- console.error(`Failed to send email notification: ${error.message}`);
2358
- }
2359
- }
2360
- /**
2361
- * Validate email address format
2362
- * @param {string} email - Email address to validate
2363
- * @returns {boolean} - True if email is valid
2364
- */
2365
- static isValidEmail(email) {
2366
- const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
2367
- const isValid = emailRegex.test(email);
2368
- if (!isValid) {
2369
- console.log(`Invalid email address skipped: ${email}`);
2370
- }
2371
- return isValid;
2372
- }
2373
- };
2374
1778
  const Core = {
2375
1779
  AbstractException,
2376
1780
  HttpRequestException,
2377
1781
  UnsupportedEnvironmentException,
2378
- EnvironmentAdapter,
2379
1782
  AbstractStorage,
2380
1783
  AbstractSource,
2381
1784
  AbstractRunConfig,
2382
1785
  AbstractConnector,
2383
1786
  AbstractConfig,
1787
+ HttpUtils,
1788
+ FileUtils,
1789
+ DateUtils,
1790
+ CryptoUtils,
1791
+ AsyncUtils,
2384
1792
  RunConfigDto,
2385
1793
  SourceConfigDto,
2386
1794
  StorageConfigDto,
2387
1795
  ConfigDto,
2388
1796
  NodeJsConfig,
2389
- GoogleSheetsConfig,
2390
- GoogleChatNotification,
2391
- EmailNotification,
2392
1797
  HTTP_STATUS,
2393
- ENVIRONMENT,
2394
1798
  EXECUTION_STATUS,
2395
1799
  RUN_CONFIG_TYPE,
2396
1800
  CONFIG_ATTRIBUTES
2397
1801
  };
2398
- const GoogleSheets = (function() {
2399
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2400
- var GoogleSheetsStorage2 = class GoogleSheetsStorage extends AbstractStorage2 {
2401
- //---- constructor -------------------------------------------------
2402
- /**
2403
- * Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
2404
- * @param config (object) instance of AbscractConfig
2405
- * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2406
- * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2407
- */
2408
- constructor(config, uniqueKeyColumns, schema = null) {
2409
- super(
2410
- config.mergeParameters({
2411
- CleanUpToKeepWindow: {
2412
- requiredType: "number"
2413
- },
2414
- DestinationSheetName: {
2415
- isRequired: true,
2416
- default: "Data"
2417
- }
2418
- }),
2419
- uniqueKeyColumns,
2420
- schema
2421
- );
2422
- this.SHEET = this.getDestinationSheet(config);
2423
- this.loadDataFromSheet();
2424
- }
2425
- //----------------------------------------------------------------
2426
- //---- loadDataFromSheet -------------------------------------------
2427
- /**
2428
- * reading Data from the source sheet and loading it to this.values
2429
- */
2430
- loadDataFromSheet() {
2431
- const values = this.SHEET.getDataRange().getValues();
2432
- this.columnNames = values.shift();
2433
- if (this.uniqueKeyColumns.some((column) => !this.columnNames.includes(column))) {
2434
- throw new Error(`Sheet '${this.SHEET.getName()}' is missing one the folling columns required for unique key: column '${this.uniqueKeyColumns}'`);
2435
- }
2436
- this.values = values.reduce((acc, row, rowIndex) => {
2437
- const uniqueKey = this.uniqueKeyColumns.reduce((accumulator, columnName) => {
2438
- let index = this.columnNames.indexOf(columnName);
2439
- accumulator += `|${row[index]}`;
2440
- return accumulator;
2441
- }, []);
2442
- acc[uniqueKey] = {
2443
- ...this.columnNames.reduce((obj, col, colIndex) => ({
2444
- ...obj,
2445
- [col]: row[colIndex]
2446
- }), {}),
2447
- rowIndex
2448
- // Add row index for reference
2449
- };
2450
- return acc;
2451
- }, {});
2452
- this.addedRecordsBuffer = [];
2453
- }
2454
- //----------------------------------------------------------------
2455
- //---- getDestinationSheet -----------------------------------------
2456
- /**
2457
- * @param object destination Spreadsheet Config
2458
- * @param object destination Sheet Name Config
2459
- * @return Sheet object for data Destination
2460
- */
2461
- getDestinationSheet(config) {
2462
- if (!this.SHEET) {
2463
- if (!config.DestinationSpreadsheet || !config.DestinationSpreadsheet.value) {
2464
- config.DestinationSpreadsheet = { "spreadsheet": config.configSpreadsheet };
2465
- } else {
2466
- let match = config.DestinationSpreadsheet.value.match(/\/d\/([a-zA-Z0-9-_]+)/);
2467
- if (match && match[1]) {
2468
- config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match[1]);
2469
- } else {
2470
- let match2 = config.DestinationSpreadsheet.cell.getRichTextValue().getLinkUrl().match(/\/d\/([a-zA-Z0-9-_]+)/);
2471
- if (match2 && match2[1]) {
2472
- config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match2[1]);
2473
- } else {
2474
- throw new Error(`Destination Spreadsheet must be specified in config either by Spreadsheet Id or by a link to spreadsheet`);
2475
- }
2476
- }
2477
- }
2478
- if (config.DestinationSpreadsheet.spreadsheet == null) {
2479
- this.config.logMessage(`Cannot load destination sheet document`);
2480
- }
2481
- config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.getSheetByName(
2482
- config.DestinationSheetName.value
2483
- );
2484
- if (!config.DestinationSpreadsheet.sheet) {
2485
- config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.insertSheet(
2486
- config.DestinationSheetName.value,
2487
- config.DestinationSpreadsheet.spreadsheet.getSheets().length
2488
- );
2489
- this.config.logMessage(`Sheet '${config.DestinationSheetName.value}' was created.`);
2490
- }
2491
- this.SHEET = config.DestinationSpreadsheet.sheet;
2492
- if (this.isEmpty()) {
2493
- this.addHeader(this.uniqueKeyColumns);
2494
- this.config.logMessage(`Columns for unique keys were added to '${config.DestinationSheetName.value}' sheet.`);
2495
- }
2496
- }
2497
- return this.SHEET;
2498
- }
2499
- //----------------------------------------------------------------
2500
- //---- addRecord ---------------------------------------------------
2501
- /**
2502
- * Checking if record exists by id
2503
- * @param object {record} object with record data
2504
- * @param Boolean {useBuffer}: Set to `false` if the record must be saved instantly, or `true` to save it later using `this.saveAddedRecordsFromBuffer()`.
2505
- * @return object {record} with added rowIndex property
2506
- */
2507
- addRecord(record, useBuffer = false) {
2508
- record = this.stringifyNeastedFields(record);
2509
- let uniqueKey = this.getUniqueKeyByRecordFields(record);
2510
- if (useBuffer) {
2511
- this.addedRecordsBuffer[uniqueKey] = record;
2512
- } else {
2513
- let data = this.columnNames.map((key) => record[key] || "");
2514
- this.SHEET.appendRow(data);
2515
- record.rowIndex = this.SHEET.getLastRow() - 2;
2516
- this.values[uniqueKey] = record;
2517
- }
2518
- return record;
2519
- }
2520
- //----------------------------------------------------------------
2521
- //---- saveData ----------------------------------------------------
2522
- /**
2523
- * Saving data to a storage
2524
- * @param {data} array of assoc objects with records to save
2525
- */
2526
- saveData(data) {
2527
- var recordsAdded = 0;
2528
- var recordsUpdated = 0;
2529
- data.map((row) => {
2530
- let newFields = Object.keys(row).filter((column2) => !this.columnNames.includes(column2));
2531
- for (var column in newFields) {
2532
- this.addColumn(newFields[column], this.columnNames.length + 1);
2533
- this.config.logMessage(`Column '${newFields[column]}' was added to '${this.SHEET.getName()}' sheet`);
2534
- }
2535
- if (this.isRecordExists(row)) {
2536
- if (this.updateRecord(row)) {
2537
- recordsUpdated++;
2538
- }
2539
- } else {
2540
- this.addRecord(row, true);
2541
- recordsAdded += this.saveRecordsAddedToBuffer(100);
2542
- }
2543
- });
2544
- recordsAdded += this.saveRecordsAddedToBuffer(0);
2545
- if (recordsAdded > 0) {
2546
- this.config.logMessage(`${recordsAdded} records were added`);
2547
- }
2548
- if (recordsUpdated > 0) {
2549
- this.config.logMessage(`${recordsUpdated} records were updated`);
2550
- }
2551
- }
2552
- //----------------------------------------------------------------
2553
- //---- saveRecordsAddedToBuffer ------------------------------------
2554
- /**
2555
- * Add records from buffer to a sheet
2556
- * @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
2557
- */
2558
- saveRecordsAddedToBuffer(maxBufferSize = 0) {
2559
- let recordsAdded = 0;
2560
- let bufferSize = Object.keys(this.addedRecordsBuffer).length;
2561
- if (bufferSize && bufferSize >= maxBufferSize) {
2562
- let startIndex = this.SHEET.getLastRow() - 2;
2563
- let index = 1;
2564
- let data = [];
2565
- for (var uniqueKey in this.addedRecordsBuffer) {
2566
- let record = this.addedRecordsBuffer[uniqueKey];
2567
- record.rowIndex = startIndex + index++;
2568
- this.values[uniqueKey] = record;
2569
- data.push(this.columnNames.map((key) => record[key] || ""));
2570
- }
2571
- this.SHEET.getRange(startIndex + 3, 1, data.length, data[0].length).setValues(data);
2572
- recordsAdded = bufferSize;
2573
- this.addedRecordsBuffer = {};
2574
- }
2575
- return recordsAdded;
2576
- }
2577
- //----------------------------------------------------------------
2578
- //---- updateRecord ------------------------------------------------
2579
- /**
2580
- * Update content of an existing record
2581
- * @param object {record} object with record data
2582
- * @return boolean Returns true if the record was updated; otherwise, returns false
2583
- */
2584
- updateRecord(record) {
2585
- record = this.stringifyNeastedFields(record);
2586
- let uniqueKey = this.getUniqueKeyByRecordFields(record);
2587
- var existingRecord = this.getRecordByUniqueKey(uniqueKey);
2588
- var isRecordUpdated = false;
2589
- this.columnNames.forEach((columnName, columnIndex) => {
2590
- if (columnName in record && !this.areValuesEqual(record[columnName], existingRecord[columnName])) {
2591
- console.log(`${uniqueKey}: ${existingRecord[columnName]} ${typeof existingRecord[columnName]} → ${record[columnName]} ${typeof record[columnName]}`);
2592
- this.SHEET.getRange(existingRecord.rowIndex + 2, columnIndex + 1, 1, 1).setValue(record[columnName]);
2593
- existingRecord[columnName] = record[columnName];
2594
- isRecordUpdated = true;
2595
- }
2596
- });
2597
- return isRecordUpdated;
2598
- }
2599
- //----------------------------------------------------------------
2600
- //---- deleteRecord ------------------------------------------------
2601
- /**
2602
- * Delete record from a sheet
2603
- * @param uniqueKey {string} unique key of the record to delete
2604
- */
2605
- deleteRecord(uniqueKey) {
2606
- if (!(uniqueKey in this.values)) {
2607
- throw new Error(`Unable to delete the record with ID ${uniqueKey} because it was not found`);
2608
- } else if (!("rowIndex" in this.values[uniqueKey])) {
2609
- throw new Error(`Unable to delete the record with ID ${uniqueKey} because it does not have a rowIndex`);
2610
- } else {
2611
- let rowIndex = this.values[uniqueKey].rowIndex;
2612
- this.SHEET.deleteRow(rowIndex + 2);
2613
- for (uniqueKey in this.values) {
2614
- if (this.values[uniqueKey].rowIndex > rowIndex) {
2615
- this.values[uniqueKey].rowIndex--;
2616
- }
2617
- }
2618
- }
2619
- }
2620
- //----------------------------------------------------------------
2621
- //---- addHeader ---------------------------------------------------
2622
- /**
2623
- * Adding header to sheet
2624
- * @param target Sheet
2625
- * @param array column names to be added
2626
- */
2627
- addHeader(columnNames) {
2628
- columnNames.forEach((columnName, index) => {
2629
- this.addColumn(columnName, index + 1);
2630
- });
2631
- this.SHEET.getRange("1:1").setBackground("#f3f3f3").setHorizontalAlignment("center");
2632
- this.SHEET.setFrozenRows(1);
2633
- }
2634
- //----------------------------------------------------------------
2635
- //---- addColumn ---------------------------------------------------
2636
- /**
2637
- * Adding a column to the sheet
2638
- * @param columnName (string) column name
2639
- * @param columnIndex (integer) optional; column index
2640
- */
2641
- addColumn(columnName, columnIndex = 1) {
2642
- const numColumns = this.SHEET.getMaxColumns();
2643
- if (columnIndex <= 0 || columnIndex > numColumns + 1) {
2644
- throw new Error(`Column index ${columnIndex} is out of bounds (1-${numColumns + 1})`);
2645
- }
2646
- if (columnIndex <= numColumns) {
2647
- const headerValue = this.SHEET.getRange(1, columnIndex).getValue();
2648
- if (headerValue !== "") {
2649
- const findFirstEmptyColumn = (startIndex) => {
2650
- let index = startIndex;
2651
- let foundEmpty = false;
2652
- while (index <= numColumns) {
2653
- if (this.SHEET.getRange(1, index).getValue() === "") {
2654
- foundEmpty = true;
2655
- break;
2656
- }
2657
- index++;
2658
- }
2659
- return {
2660
- columnIndex: index,
2661
- foundEmpty
2662
- };
2663
- };
2664
- const result = findFirstEmptyColumn(columnIndex);
2665
- if (!result.foundEmpty) {
2666
- this.SHEET.insertColumnAfter(numColumns);
2667
- columnIndex = numColumns + 1;
2668
- } else {
2669
- this.SHEET.insertColumnBefore(result.columnIndex);
2670
- columnIndex = result.columnIndex;
2671
- }
2672
- }
2673
- } else {
2674
- this.SHEET.insertColumnAfter(numColumns);
2675
- columnIndex = numColumns + 1;
2676
- }
2677
- this.SHEET.getRange(1, columnIndex).setValue(columnName);
2678
- if (this.schema != null && columnName in this.schema && "GoogleSheetsFormat" in this.schema[columnName]) {
2679
- let columnLetter = String.fromCharCode(64 + columnIndex);
2680
- console.log(
2681
- columnName,
2682
- this.schema[columnName]["GoogleSheetsFormat"],
2683
- this.SHEET.getRange(`${columnLetter}:${columnLetter}`).getA1Notation()
2684
- );
2685
- this.SHEET.getRange(`${columnLetter}:${columnLetter}`).setNumberFormat(this.schema[columnName]["GoogleSheetsFormat"]);
2686
- }
2687
- this.columnNames.push(columnName);
2688
- }
2689
- //----------------------------------------------------------------
2690
- //---- formatColumn ------------------------------------------------
2691
- /**
2692
- * Format column as it described in schema
2693
- * @param columnName (string) column name
2694
- */
2695
- formatColumn(columnName) {
2696
- if ("type" in this.schema[columnName]) ;
2697
- }
2698
- //----------------------------------------------------------------
2699
- //---- getColumnIndexByName ----------------------------------------
2700
- /**
2701
- * @param columnName string column name
2702
- * @return integer columnIndex
2703
- */
2704
- getColumnIndexByName(columnName) {
2705
- const columnIndex = this.columnNames.indexOf(columnName);
2706
- if (columnIndex == -1) {
2707
- throw new Error(`Column ${columnName} not found in '${this.SHEET.getName()}' sheet`);
2708
- }
2709
- return columnIndex + 1;
2710
- }
2711
- //----------------------------------------------------------------
2712
- //---- isEmpty -----------------------------------------------------
2713
- /**
2714
- * @return boolean true if sheet is empty, false overwise
2715
- */
2716
- isEmpty() {
2717
- return this.SHEET.getLastRow() === 0 && this.SHEET.getLastColumn() === 0;
2718
- }
2719
- //----------------------------------------------------------------
2720
- //---- areValuesEqual ----------------------------------------------
2721
- /**
2722
- * Comparing to vaariables if they are equal or not
2723
- * @param value1 (mixed)
2724
- * @param value2 (mixed)
2725
- * @return boolean true if equal, falst overwise
2726
- */
2727
- areValuesEqual(value1, value2) {
2728
- var equal = null;
2729
- if (typeof value1 !== "undefined" && typeof value2 !== "undefined" && ((value1 == null ? void 0 : value1.constructor.name) == "Date" || (value2 == null ? void 0 : value2.constructor.name) == "Date")) {
2730
- const normalizeToDate = (value) => {
2731
- if (value === null || value === "") return null;
2732
- if (value.constructor.name == "Date") return value;
2733
- const date = new Date(value);
2734
- return isNaN(date.getTime()) ? null : date;
2735
- };
2736
- const date1 = normalizeToDate(value1);
2737
- const date2 = normalizeToDate(value2);
2738
- if (date1 === null || date2 === null) {
2739
- return value1 === value2;
2740
- }
2741
- equal = date1.getTime() === date2.getTime();
2742
- } else if (typeof value1 == typeof value2) {
2743
- equal = value1 === value2;
2744
- } else if (value1 === void 0 && value2 === "" || value2 === void 0 && value1 === "") {
2745
- equal = true;
2746
- } else if (value1 === null && value2 === "" || value2 === null && value1 === "") {
2747
- equal = true;
2748
- }
2749
- return equal;
2750
- }
2751
- //----------------------------------------------------------------
2752
- //---- areHeadersNeeded ------------------------------------------
2753
- /**
2754
- * Checks if storage is empty and adds headers if needed
2755
- * if destination sheet is empty than header should be created based on unique key columns list
2756
- * @return {boolean} true if headers were added, false if they already existed
2757
- */
2758
- areHeadersNeeded() {
2759
- return this.isEmpty();
2760
- }
2761
- //----------------------------------------------------------------
2762
- };
2763
- const manifest = {
2764
- "name": "GoogleSheetsStorage",
2765
- "description": "Storage for Google Sheets",
2766
- "title": "Google Sheets",
2767
- "version": "0.0.0",
2768
- "author": "OWOX, Inc.",
2769
- "license": "MIT",
2770
- "environment": {
2771
- "node": {
2772
- "enabled": false
2773
- },
2774
- "appscript": {
2775
- "enabled": true
2776
- }
2777
- }
2778
- };
2779
- return {
2780
- GoogleSheetsStorage: GoogleSheetsStorage2,
2781
- manifest
2782
- };
2783
- })();
2784
1802
  const GoogleBigQuery = (function() {
2785
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
1803
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2786
1804
  var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
2787
1805
  //---- constructor -------------------------------------------------
2788
1806
  /**
2789
- * Asbstract class making Google BigQuery updateable in Apps Script
2790
- *
2791
- * @param config (object) instance of AbscractConfig
1807
+ * Abstract class for Google BigQuery storage operations
1808
+ *
1809
+ * @param config (object) instance of AbstractConfig
2792
1810
  * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2793
1811
  * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2794
1812
  * @param description (string) string with storage description }
@@ -2834,22 +1852,29 @@ ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}
2834
1852
  schema,
2835
1853
  description
2836
1854
  );
2837
- this.checkIfGoogleBigQueryIsConnected();
2838
- this.loadTableSchema();
2839
1855
  this.updatedRecordsBuffer = {};
2840
1856
  this.totalRecordsProcessed = 0;
2841
1857
  }
1858
+ //---- init --------------------------------------------------------
1859
+ /**
1860
+ * Initializing storage
1861
+ */
1862
+ async init() {
1863
+ this.checkIfGoogleBigQueryIsConnected();
1864
+ await this.loadTableSchema();
1865
+ }
1866
+ //----------------------------------------------------------------
2842
1867
  //---- loads Google BigQuery Table Schema ---------------------------
2843
- loadTableSchema() {
1868
+ async loadTableSchema() {
2844
1869
  this.existingColumns = this.getAListOfExistingColumns() || {};
2845
1870
  if (Object.keys(this.existingColumns).length == 0) {
2846
- this.createDatasetIfItDoesntExist();
2847
- this.existingColumns = this.createTableIfItDoesntExist();
1871
+ await this.createDatasetIfItDoesntExist();
1872
+ this.existingColumns = await this.createTableIfItDoesntExist();
2848
1873
  } else {
2849
1874
  let selectedFields = this.getSelectedFields();
2850
1875
  let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
2851
1876
  if (newFields.length > 0) {
2852
- this.addNewColumns(newFields);
1877
+ await this.addNewColumns(newFields);
2853
1878
  }
2854
1879
  }
2855
1880
  }
@@ -2887,17 +1912,17 @@ ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}
2887
1912
  return columns;
2888
1913
  }
2889
1914
  //---- createDatasetIfItDoesntExist --------------------------------
2890
- createDatasetIfItDoesntExist() {
1915
+ async createDatasetIfItDoesntExist() {
2891
1916
  let query = `---- Create Dataset if it not exists -----
2892
1917
  `;
2893
1918
  query += `CREATE SCHEMA IF NOT EXISTS \`${this.config.DestinationProjectID.value}.${this.config.DestinationDatasetName.value}\`
2894
1919
  OPTIONS (
2895
1920
  location = '${this.config.DestinationLocation.value}'
2896
1921
  )`;
2897
- this.executeQuery(query);
1922
+ await this.executeQuery(query);
2898
1923
  }
2899
1924
  //---- createTableIfItDoesntExist ----------------------------------
2900
- createTableIfItDoesntExist() {
1925
+ async createTableIfItDoesntExist() {
2901
1926
  let columns = [];
2902
1927
  let columnPartitioned = null;
2903
1928
  let existingColumns = {};
@@ -2933,15 +1958,14 @@ PARTITION BY ${columnPartitioned}`;
2933
1958
  query += `
2934
1959
  OPTIONS(description="${this.description}")`;
2935
1960
  }
2936
- this.executeQuery(query);
1961
+ await this.executeQuery(query);
2937
1962
  this.config.logMessage(`Table ${this.config.DestinationDatasetID.value}.${this.config.DestinationTableName.value} was created`);
2938
1963
  return existingColumns;
2939
1964
  }
2940
1965
  //---- checkIfGoogleBigQueryIsConnected ---------------------
2941
1966
  checkIfGoogleBigQueryIsConnected() {
2942
1967
  if (typeof BigQuery == "undefined") {
2943
- throw new Error(`To import data into Google BigQuery you need to add BigQuery Service first:
2944
- Extension / Apps Script / Editor / Services / + BigQuery API`);
1968
+ throw new Error(`BigQuery client library is not available. Ensure @google-cloud/bigquery is installed.`);
2945
1969
  }
2946
1970
  }
2947
1971
  //---- addNewColumns -----------------------------------------------
@@ -2952,7 +1976,7 @@ OPTIONS(description="${this.description}")`;
2952
1976
  * @param {newColumns} array with a list of new columns
2953
1977
  *
2954
1978
  */
2955
- addNewColumns(newColumns) {
1979
+ async addNewColumns(newColumns) {
2956
1980
  let query = "";
2957
1981
  let columns = [];
2958
1982
  for (var i in newColumns) {
@@ -2974,7 +1998,7 @@ OPTIONS(description="${this.description}")`;
2974
1998
 
2975
1999
  `;
2976
2000
  query += columns.join(",\n");
2977
- this.executeQuery(query);
2001
+ await this.executeQuery(query);
2978
2002
  this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.DestinationDatasetID.value} dataset`);
2979
2003
  }
2980
2004
  }
@@ -2983,17 +2007,17 @@ OPTIONS(description="${this.description}")`;
2983
2007
  * Saving data to a storage
2984
2008
  * @param {data} array of assoc objects with records to save
2985
2009
  */
2986
- saveData(data) {
2987
- data.map((row) => {
2010
+ async saveData(data) {
2011
+ for (const row of data) {
2988
2012
  let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
2989
2013
  if (newFields.length > 0) {
2990
2014
  console.log(newFields);
2991
- this.addNewColumns(newFields);
2015
+ await this.addNewColumns(newFields);
2992
2016
  }
2993
2017
  this.addRecordToBuffer(row);
2994
- this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2995
- });
2996
- this.saveRecordsAddedToBuffer();
2018
+ await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2019
+ }
2020
+ await this.saveRecordsAddedToBuffer();
2997
2021
  }
2998
2022
  // ------- addReordTuBuffer ---------------------
2999
2023
  /**
@@ -3008,24 +2032,24 @@ OPTIONS(description="${this.description}")`;
3008
2032
  * Add records from buffer to a sheet
3009
2033
  * @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
3010
2034
  */
3011
- saveRecordsAddedToBuffer(maxBufferSize = 0) {
2035
+ async saveRecordsAddedToBuffer(maxBufferSize = 0) {
3012
2036
  let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
3013
2037
  if (bufferSize && bufferSize >= maxBufferSize) {
3014
2038
  console.log(`Starting BigQuery MERGE operation for ${bufferSize} records...`);
3015
- this.executeQueryWithSizeLimit();
2039
+ await this.executeQueryWithSizeLimit();
3016
2040
  }
3017
2041
  }
3018
2042
  //---- executeQueryWithSizeLimit ----------------------------------
3019
2043
  /**
3020
2044
  * Executes the MERGE query with automatic size reduction if it exceeds BigQuery limits
3021
2045
  */
3022
- executeQueryWithSizeLimit() {
2046
+ async executeQueryWithSizeLimit() {
3023
2047
  const bufferKeys = Object.keys(this.updatedRecordsBuffer);
3024
2048
  const totalRecords = bufferKeys.length;
3025
2049
  if (totalRecords === 0) {
3026
2050
  return;
3027
2051
  }
3028
- this.executeMergeQueryRecursively(bufferKeys, totalRecords);
2052
+ await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
3029
2053
  this.updatedRecordsBuffer = {};
3030
2054
  }
3031
2055
  //---- executeMergeQueryRecursively --------------------------------
@@ -3034,7 +2058,7 @@ OPTIONS(description="${this.description}")`;
3034
2058
  * @param {Array} recordKeys - Array of record keys to process
3035
2059
  * @param {number} batchSize - Current batch size to attempt
3036
2060
  */
3037
- executeMergeQueryRecursively(recordKeys, batchSize) {
2061
+ async executeMergeQueryRecursively(recordKeys, batchSize) {
3038
2062
  if (recordKeys.length === 0) {
3039
2063
  return;
3040
2064
  }
@@ -3048,11 +2072,11 @@ OPTIONS(description="${this.description}")`;
3048
2072
  const maxQuerySize = 1024 * 1024;
3049
2073
  if (querySize > maxQuerySize) {
3050
2074
  console.log(`Query size (${Math.round(querySize / 1024)}KB) exceeds BigQuery limit. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
3051
- this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
2075
+ await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
3052
2076
  return;
3053
2077
  }
3054
2078
  try {
3055
- this.executeQuery(query);
2079
+ await this.executeQuery(query);
3056
2080
  this.totalRecordsProcessed += currentBatch.length;
3057
2081
  console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
3058
2082
  if (remainingRecords.length > 0) {
@@ -3086,9 +2110,10 @@ OPTIONS(description="${this.description}")`;
3086
2110
  if (record[columnName] === void 0 || record[columnName] === null) {
3087
2111
  columnValue = null;
3088
2112
  } else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
3089
- columnValue = EnvironmentAdapter2.formatDate(record[columnName], "UTC", "yyyy-MM-dd");
2113
+ columnValue = DateUtils2.formatDate(record[columnName]);
3090
2114
  } else if (columnType.toUpperCase() == "DATETIME" && record[columnName] instanceof Date) {
3091
- columnValue = EnvironmentAdapter2.formatDate(record[columnName], "UTC", "yyyy-MM-dd HH:mm:ss");
2115
+ const isoString = record[columnName].toISOString();
2116
+ columnValue = isoString.replace("T", " ").substring(0, 19);
3092
2117
  } else {
3093
2118
  columnValue = this.obfuscateSpecialCharacters(record[columnName]);
3094
2119
  }
@@ -3123,53 +2148,36 @@ OPTIONS(description="${this.description}")`;
3123
2148
  //---- query -------------------------------------------------------
3124
2149
  /**
3125
2150
  * Executes Google BigQuery Query and returns a result
3126
- *
3127
- * @param {query} string
3128
- *
3129
- * @return object
3130
- *
2151
+ *
2152
+ * @param {query} string
2153
+ *
2154
+ * @return Promise<object>
2155
+ *
3131
2156
  */
3132
- executeQuery(query) {
3133
- if (this.config.Environment.value === ENVIRONMENT2.APPS_SCRIPT) {
3134
- return BigQuery.Jobs.query(
3135
- { "query": query, useLegacySql: false },
3136
- this.config.ProjectID.value
3137
- );
3138
- }
3139
- if (this.config.Environment.value === ENVIRONMENT2.NODE) {
3140
- let result = void 0;
3141
- let error = void 0;
3142
- let bigqueryClient = null;
3143
- if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
3144
- const { JWT } = require("google-auth-library");
3145
- const credentials = JSON.parse(this.config.ServiceAccountJson.value);
3146
- const authClient = new JWT({
3147
- email: credentials.client_email,
3148
- key: credentials.private_key,
3149
- scopes: ["https://www.googleapis.com/auth/bigquery"]
3150
- });
3151
- bigqueryClient = new BigQuery({
3152
- projectId: this.config.ProjectID.value || credentials.project_id,
3153
- authClient
3154
- });
3155
- } else {
3156
- throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
3157
- }
3158
- const options = {
3159
- query,
3160
- useLegacySql: false
3161
- };
3162
- bigqueryClient.createQueryJob(options).then(([job]) => job.getQueryResults()).then(([rows]) => rows).then((value) => {
3163
- result = value;
3164
- }).catch((e) => {
3165
- error = e;
2157
+ async executeQuery(query) {
2158
+ let bigqueryClient = null;
2159
+ if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
2160
+ const { JWT } = require("google-auth-library");
2161
+ const credentials = JSON.parse(this.config.ServiceAccountJson.value);
2162
+ const authClient = new JWT({
2163
+ email: credentials.client_email,
2164
+ key: credentials.private_key,
2165
+ scopes: ["https://www.googleapis.com/auth/bigquery"]
3166
2166
  });
3167
- deasync.loopWhile(() => result === void 0 && error === void 0);
3168
- if (error !== void 0) {
3169
- throw error;
3170
- }
3171
- return result;
2167
+ bigqueryClient = new BigQuery({
2168
+ projectId: this.config.ProjectID.value || credentials.project_id,
2169
+ authClient
2170
+ });
2171
+ } else {
2172
+ throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
3172
2173
  }
2174
+ const options = {
2175
+ query,
2176
+ useLegacySql: false
2177
+ };
2178
+ const [job] = await bigqueryClient.createQueryJob(options);
2179
+ const [rows] = await job.getQueryResults();
2180
+ return rows;
3173
2181
  }
3174
2182
  //---- obfuscateSpecialCharacters ----------------------------------
3175
2183
  obfuscateSpecialCharacters(inputString) {
@@ -3265,7 +2273,7 @@ OPTIONS(description="${this.description}")`;
3265
2273
  };
3266
2274
  })();
3267
2275
  const AwsAthena = (function() {
3268
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2276
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
3269
2277
  var AwsAthenaStorage = class AwsAthenaStorage extends AbstractStorage2 {
3270
2278
  //---- constructor -------------------------------------------------
3271
2279
  /**
@@ -3323,9 +2331,21 @@ OPTIONS(description="${this.description}")`;
3323
2331
  this.initAWS();
3324
2332
  this.updatedRecordsBuffer = {};
3325
2333
  this.existingColumns = {};
3326
- this.setupAthenaDatabase();
3327
2334
  this.uploadSid = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:.]/g, "") + "_" + Math.random().toString(36).substring(2, 15);
3328
2335
  }
2336
+ //---- init --------------------------------------------------------
2337
+ /**
2338
+ * Initializing storage
2339
+ */
2340
+ async init() {
2341
+ const success = await this.setupAthenaDatabase();
2342
+ if (success) {
2343
+ console.log("Database created or already exists");
2344
+ } else {
2345
+ throw new Error("Failed to create database");
2346
+ }
2347
+ }
2348
+ //----------------------------------------------------------------
3329
2349
  //---- initAWS ----------------------------------------------------
3330
2350
  /**
3331
2351
  * Initialize AWS SDK clients
@@ -3357,61 +2377,58 @@ OPTIONS(description="${this.description}")`;
3357
2377
  /**
3358
2378
  * Create Athena database if it doesn't exist
3359
2379
  */
3360
- createDatabaseIfNotExists() {
2380
+ async createDatabaseIfNotExists() {
3361
2381
  const params = {
3362
2382
  QueryString: `CREATE SCHEMA IF NOT EXISTS \`${this.config.AthenaDatabaseName.value}\``,
3363
2383
  ResultConfiguration: {
3364
2384
  OutputLocation: this.config.AthenaOutputLocation.value
3365
2385
  }
3366
2386
  };
3367
- return this.executeQuery(params, "ddl").then(() => {
3368
- this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
3369
- return true;
3370
- });
2387
+ await this.executeQuery(params, "ddl");
2388
+ this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
2389
+ return true;
3371
2390
  }
3372
2391
  //---- checkTableExists ------------------------------------------
3373
2392
  /**
3374
2393
  * Check if the target table exists in Athena
3375
2394
  */
3376
- checkTableExists() {
2395
+ async checkTableExists() {
3377
2396
  const params = {
3378
2397
  QueryString: `SHOW TABLES IN \`${this.config.AthenaDatabaseName.value}\` LIKE '${this.config.DestinationTableName.value}'`,
3379
2398
  ResultConfiguration: {
3380
2399
  OutputLocation: this.config.AthenaOutputLocation.value
3381
2400
  }
3382
2401
  };
3383
- return this.executeQuery(params, "ddl").then((results) => {
2402
+ try {
2403
+ const results = await this.executeQuery(params, "ddl");
3384
2404
  if (results && results.length > 0) {
3385
- return this.getTableSchema();
2405
+ return await this.getTableSchema();
3386
2406
  }
3387
- return this.createTargetTable();
3388
- }).catch(() => {
3389
- return this.createTargetTable();
3390
- });
2407
+ return await this.createTargetTable();
2408
+ } catch {
2409
+ return await this.createTargetTable();
2410
+ }
3391
2411
  }
3392
2412
  //---- getTableSchema -------------------------------------------
3393
2413
  /**
3394
2414
  * Get the schema of the existing table
3395
2415
  */
3396
- getTableSchema() {
2416
+ async getTableSchema() {
3397
2417
  const params = {
3398
2418
  QueryString: `SHOW COLUMNS IN \`${this.config.AthenaDatabaseName.value}\`.\`${this.config.DestinationTableName.value}\``,
3399
2419
  ResultConfiguration: {
3400
2420
  OutputLocation: this.config.AthenaOutputLocation.value
3401
2421
  }
3402
2422
  };
3403
- return this.executeQuery(params, "ddl").then((results) => {
3404
- let columns = {};
3405
- if (results && results.length > 0) {
3406
- results.forEach((row) => {
3407
- columns[row] = this.getColumnType(row);
3408
- });
3409
- }
3410
- this.existingColumns = columns;
3411
- return columns;
3412
- }).catch((error) => {
3413
- return {};
3414
- });
2423
+ const results = await this.executeQuery(params, "ddl");
2424
+ let columns = {};
2425
+ if (results && results.length > 0) {
2426
+ results.forEach((row) => {
2427
+ columns[row] = this.getColumnType(row);
2428
+ });
2429
+ }
2430
+ this.existingColumns = columns;
2431
+ return columns;
3415
2432
  }
3416
2433
  //---- createTargetTable ----------------------------------------------
3417
2434
  /**
@@ -3501,45 +2518,42 @@ OPTIONS(description="${this.description}")`;
3501
2518
  * @param {Array} data - Array of objects with records to save
3502
2519
  * @returns {Promise}
3503
2520
  */
3504
- saveData(data) {
3505
- let done = false;
3506
- this.checkTableExists().then(() => {
3507
- const allColumns = /* @__PURE__ */ new Set();
3508
- if (data.length > 0) {
3509
- data.forEach((row) => {
3510
- Object.keys(row).forEach((column) => allColumns.add(column));
3511
- });
3512
- }
3513
- if (this.config.Fields.value) {
3514
- this.getSelectedFields().forEach((columnName) => {
3515
- if (columnName && !allColumns.has(columnName)) {
3516
- allColumns.add(columnName);
3517
- if (data.length > 0) {
3518
- data.forEach((row) => {
3519
- if (!row[columnName]) {
3520
- row[columnName] = "";
3521
- }
3522
- });
3523
- }
2521
+ async saveData(data) {
2522
+ await this.checkTableExists();
2523
+ const allColumns = /* @__PURE__ */ new Set();
2524
+ if (data.length > 0) {
2525
+ data.forEach((row) => {
2526
+ Object.keys(row).forEach((column) => allColumns.add(column));
2527
+ });
2528
+ }
2529
+ if (this.config.Fields.value) {
2530
+ this.getSelectedFields().forEach((columnName) => {
2531
+ if (columnName && !allColumns.has(columnName)) {
2532
+ allColumns.add(columnName);
2533
+ if (data.length > 0) {
2534
+ data.forEach((row) => {
2535
+ if (!row[columnName]) {
2536
+ row[columnName] = "";
2537
+ }
2538
+ });
3524
2539
  }
3525
- });
3526
- }
3527
- const existingColumnsSet = new Set(Object.keys(this.existingColumns));
3528
- const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
3529
- if (newColumns.length > 0) {
3530
- return this.addNewColumns(newColumns);
3531
- }
3532
- return Promise.resolve();
3533
- }).then(() => {
3534
- if (data.length === 0) {
3535
- done = true;
3536
- return;
3537
- }
3538
- this.config.logMessage(`Saving ${data.length} records to Athena`);
3539
- const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
3540
- return this.uploadDataToS3TempFolder(data, tempFolder).then(() => this.createTempTable(tempFolder, this.uploadSid)).then((tempTableName) => this.mergeDataFromTempTable(tempTableName, this.uploadSid)).then((tempTableName) => this.cleanupTempResources(tempFolder, tempTableName)).then(() => done = true);
3541
- });
3542
- deasync.loopWhile(() => !done);
2540
+ }
2541
+ });
2542
+ }
2543
+ const existingColumnsSet = new Set(Object.keys(this.existingColumns));
2544
+ const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
2545
+ if (newColumns.length > 0) {
2546
+ await this.addNewColumns(newColumns);
2547
+ }
2548
+ if (data.length === 0) {
2549
+ return;
2550
+ }
2551
+ this.config.logMessage(`Saving ${data.length} records to Athena`);
2552
+ const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
2553
+ await this.uploadDataToS3TempFolder(data, tempFolder);
2554
+ const tempTableName = await this.createTempTable(tempFolder, this.uploadSid);
2555
+ await this.mergeDataFromTempTable(tempTableName, this.uploadSid);
2556
+ await this.cleanupTempResources(tempFolder, tempTableName);
3543
2557
  }
3544
2558
  //---- uploadDataToS3TempFolder ---------------------------------
3545
2559
  /**
@@ -3728,6 +2742,7 @@ OPTIONS(description="${this.description}")`;
3728
2742
  QueryExecutionId: queryExecutionId
3729
2743
  };
3730
2744
  return this.athenaClient.send(new GetQueryExecutionCommand(params)).then((data) => {
2745
+ var _a;
3731
2746
  const state = data.QueryExecution.Status.State;
3732
2747
  if (state === "SUCCEEDED") {
3733
2748
  return new Promise((resolve) => {
@@ -3736,6 +2751,7 @@ OPTIONS(description="${this.description}")`;
3736
2751
  }, 3e3);
3737
2752
  });
3738
2753
  } else if (state === "FAILED" || state === "CANCELLED") {
2754
+ this.config.logMessage(`Query ${queryExecutionId} ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}. Error: ${((_a = data.QueryExecution.Status.Error) == null ? void 0 : _a.Message) || ""}`);
3739
2755
  throw new Error(`Query ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}`);
3740
2756
  } else {
3741
2757
  return new Promise((resolve) => {
@@ -3895,12 +2911,11 @@ OPTIONS(description="${this.description}")`;
3895
2911
  };
3896
2912
  })();
3897
2913
  const Storages = {
3898
- GoogleSheets,
3899
2914
  GoogleBigQuery,
3900
2915
  AwsAthena
3901
2916
  };
3902
2917
  const XAds = (function() {
3903
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
2918
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
3904
2919
  const XAdsHelper = {
3905
2920
  /**
3906
2921
  * Parse fields string into a structured object
@@ -4794,34 +3809,34 @@ OPTIONS(description="${this.description}")`;
4794
3809
  * @param {string} [opts.end_time]
4795
3810
  * @returns {Array<Object>}
4796
3811
  */
4797
- fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
4798
- EnvironmentAdapter2.sleep(this.config.AdsApiDelay.value * 1e3);
3812
+ async fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
3813
+ await AsyncUtils2.delay(this.config.AdsApiDelay.value * 1e3);
4799
3814
  switch (nodeName) {
4800
3815
  case "accounts": {
4801
- const resp = this._getData(`accounts/${accountId}`, "accounts", fields);
3816
+ const resp = await this._getData(`accounts/${accountId}`, "accounts", fields);
4802
3817
  return [resp.data];
4803
3818
  }
4804
3819
  case "campaigns":
4805
3820
  case "line_items":
4806
3821
  case "promoted_tweets":
4807
3822
  case "tweets":
4808
- return this._catalogFetch({
3823
+ return await this._catalogFetch({
4809
3824
  nodeName,
4810
3825
  accountId,
4811
3826
  fields,
4812
3827
  pageSize: this.config.DataMaxCount.value
4813
3828
  });
4814
3829
  case "cards":
4815
- return this._catalogFetch({
3830
+ return await this._catalogFetch({
4816
3831
  nodeName,
4817
3832
  accountId,
4818
3833
  fields,
4819
3834
  pageSize: this.config.CardsMaxCountPerRequest.value
4820
3835
  });
4821
3836
  case "cards_all":
4822
- return this._fetchAllCards(accountId, fields);
3837
+ return await this._fetchAllCards(accountId, fields);
4823
3838
  case "stats":
4824
- return this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
3839
+ return await this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
4825
3840
  default:
4826
3841
  throw new ConfigurationError(`Unknown node: ${nodeName}`);
4827
3842
  }
@@ -4857,7 +3872,7 @@ OPTIONS(description="${this.description}")`;
4857
3872
  /**
4858
3873
  * Shared logic for non-time-series endpoints
4859
3874
  */
4860
- _catalogFetch({ nodeName, accountId, fields, pageSize }) {
3875
+ async _catalogFetch({ nodeName, accountId, fields, pageSize }) {
4861
3876
  const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
4862
3877
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
4863
3878
  if (missingKeys.length > 0) {
@@ -4881,7 +3896,7 @@ OPTIONS(description="${this.description}")`;
4881
3896
  console.log("deleting cached tweets");
4882
3897
  this._tweetsCache.delete(accountId);
4883
3898
  }
4884
- let all = this._fetchPages({
3899
+ let all = await this._fetchPages({
4885
3900
  accountId,
4886
3901
  nodeName,
4887
3902
  fields,
@@ -4905,7 +3920,7 @@ OPTIONS(description="${this.description}")`;
4905
3920
  /**
4906
3921
  * Shared pagination logic
4907
3922
  */
4908
- _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
3923
+ async _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
4909
3924
  const all = [];
4910
3925
  let cursor = null;
4911
3926
  const MAX_PAGES = 100;
@@ -4916,7 +3931,7 @@ OPTIONS(description="${this.description}")`;
4916
3931
  ...extraParams,
4917
3932
  ...cursor ? { cursor } : {}
4918
3933
  };
4919
- const resp = this._getData(
3934
+ const resp = await this._getData(
4920
3935
  `accounts/${accountId}/${nodeName}`,
4921
3936
  nodeName,
4922
3937
  fields,
@@ -4937,15 +3952,15 @@ OPTIONS(description="${this.description}")`;
4937
3952
  * Fetch all cards by first collecting URIs from tweets,
4938
3953
  * then calling the cards/all endpoint in chunks.
4939
3954
  */
4940
- _fetchAllCards(accountId, fields) {
4941
- const tweets = this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
3955
+ async _fetchAllCards(accountId, fields) {
3956
+ const tweets = await this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
4942
3957
  const uris = tweets.map((t) => t.card_uri).filter(Boolean);
4943
3958
  if (!uris.length) return [];
4944
3959
  const all = [];
4945
3960
  const chunkSize = this.config.CardsMaxCountPerRequest.value;
4946
3961
  for (let i = 0; i < uris.length; i += chunkSize) {
4947
3962
  const chunk = uris.slice(i, i + chunkSize);
4948
- const resp = this._getData(
3963
+ const resp = await this._getData(
4949
3964
  `accounts/${accountId}/cards/all`,
4950
3965
  "cards_all",
4951
3966
  fields,
@@ -4962,18 +3977,18 @@ OPTIONS(description="${this.description}")`;
4962
3977
  /**
4963
3978
  * Stats are time-series and need flattening of `metrics`
4964
3979
  */
4965
- _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
3980
+ async _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
4966
3981
  const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
4967
3982
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
4968
3983
  if (missingKeys.length > 0) {
4969
3984
  throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
4970
3985
  }
4971
- const promos = this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
3986
+ const promos = await this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
4972
3987
  const ids = promos.map((r) => r.id);
4973
3988
  if (!ids.length) return [];
4974
3989
  const e = new Date(end_time);
4975
3990
  e.setDate(e.getDate() + 1);
4976
- const endStr = EnvironmentAdapter2.formatDate(e, "UTC", "yyyy-MM-dd");
3991
+ const endStr = DateUtils2.formatDate(e);
4977
3992
  const result = [];
4978
3993
  for (let i = 0; i < ids.length; i += this.config.StatsMaxEntityIds.value) {
4979
3994
  const batch = ids.slice(i, i + this.config.StatsMaxEntityIds.value).join(",");
@@ -4986,7 +4001,7 @@ OPTIONS(description="${this.description}")`;
4986
4001
  end_time: endStr
4987
4002
  };
4988
4003
  for (const placement of ["ALL_ON_TWITTER", "PUBLISHER_NETWORK"]) {
4989
- const raw = this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
4004
+ const raw = await this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
4990
4005
  const arr = Array.isArray(raw.data) ? raw.data : [raw.data];
4991
4006
  arr.forEach((h) => {
4992
4007
  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
@@ -5022,18 +4037,19 @@ OPTIONS(description="${this.description}")`;
5022
4037
  /**
5023
4038
  * Pull JSON from the Ads API (raw, no field-filter).
5024
4039
  */
5025
- _rawFetch(path, params = {}) {
4040
+ async _rawFetch(path, params = {}) {
5026
4041
  const url = `${this.BASE_URL}${this.config.Version.value}/${path}`;
5027
4042
  const qs = Object.keys(params).length ? "?" + Object.entries(params).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
5028
4043
  const finalUrl = url + qs;
5029
4044
  const oauth = this._generateOAuthHeader({ method: "GET", url, params });
5030
- EnvironmentAdapter2.sleep(1e3);
5031
- const resp = this.urlFetchWithRetry(finalUrl, {
4045
+ await AsyncUtils2.delay(1e3);
4046
+ const resp = await this.urlFetchWithRetry(finalUrl, {
5032
4047
  method: "GET",
5033
4048
  headers: { Authorization: oauth, "Content-Type": "application/json" },
5034
4049
  muteHttpExceptions: true
5035
4050
  });
5036
- return JSON.parse(resp.getContentText());
4051
+ const text = await resp.getContentText();
4052
+ return JSON.parse(text);
5037
4053
  }
5038
4054
  /**
5039
4055
  * Determines if a X Ads API error is valid for retry
@@ -5056,8 +4072,8 @@ OPTIONS(description="${this.description}")`;
5056
4072
  }
5057
4073
  return false;
5058
4074
  }
5059
- _getData(path, nodeName, fields, extraParams = {}) {
5060
- const json = this._rawFetch(path, extraParams);
4075
+ async _getData(path, nodeName, fields, extraParams = {}) {
4076
+ const json = await this._rawFetch(path, extraParams);
5061
4077
  if (!json.data) return json;
5062
4078
  const arr = Array.isArray(json.data) ? json.data : [json.data];
5063
4079
  const filtered = this._filterBySchema(arr, nodeName, fields);
@@ -5096,7 +4112,7 @@ OPTIONS(description="${this.description}")`;
5096
4112
  const { ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret } = this.config;
5097
4113
  const oauth = {
5098
4114
  oauth_consumer_key: ConsumerKey.value,
5099
- oauth_nonce: EnvironmentAdapter2.getUuid().replace(/-/g, ""),
4115
+ oauth_nonce: CryptoUtils2.getUuid().replace(/-/g, ""),
5100
4116
  oauth_signature_method: "HMAC-SHA1",
5101
4117
  oauth_timestamp: Math.floor(Date.now() / 1e3),
5102
4118
  oauth_token: AccessToken.value,
@@ -5111,9 +4127,9 @@ OPTIONS(description="${this.description}")`;
5111
4127
  )
5112
4128
  ].join("&");
5113
4129
  const signingKey = encodeURIComponent(ConsumerSecret.value) + "&" + encodeURIComponent(AccessTokenSecret.value);
5114
- oauth.oauth_signature = EnvironmentAdapter2.base64Encode(
5115
- EnvironmentAdapter2.computeHmacSignature(
5116
- EnvironmentAdapter2.MacAlgorithm.HMAC_SHA_1,
4130
+ oauth.oauth_signature = CryptoUtils2.base64Encode(
4131
+ CryptoUtils2.computeHmacSignature(
4132
+ CryptoUtils2.MacAlgorithm.HMAC_SHA_1,
5117
4133
  baseString,
5118
4134
  signingKey
5119
4135
  )
@@ -5129,7 +4145,7 @@ OPTIONS(description="${this.description}")`;
5129
4145
  }
5130
4146
  };
5131
4147
  var XAdsConnector = class XAdsConnector extends AbstractConnector2 {
5132
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
4148
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
5133
4149
  super(config, source, null, runConfig);
5134
4150
  this.storageName = storageName;
5135
4151
  }
@@ -5137,12 +4153,12 @@ OPTIONS(description="${this.description}")`;
5137
4153
  * Main method - entry point for the import process
5138
4154
  * Processes all nodes defined in the fields configuration
5139
4155
  */
5140
- startImportProcess() {
4156
+ async startImportProcess() {
5141
4157
  const fields = XAdsHelper.parseFields(this.config.Fields.value);
5142
4158
  const accountIds = XAdsHelper.parseAccountIds(this.config.AccountIDs.value);
5143
4159
  for (const accountId of accountIds) {
5144
4160
  for (const nodeName in fields) {
5145
- this.processNode({
4161
+ await this.processNode({
5146
4162
  nodeName,
5147
4163
  accountId,
5148
4164
  fields: fields[nodeName] || []
@@ -5158,15 +4174,15 @@ OPTIONS(description="${this.description}")`;
5158
4174
  * @param {string} options.accountId - Account ID
5159
4175
  * @param {Array<string>} options.fields - Array of fields to fetch
5160
4176
  */
5161
- processNode({ nodeName, accountId, fields }) {
4177
+ async processNode({ nodeName, accountId, fields }) {
5162
4178
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
5163
- this.processTimeSeriesNode({
4179
+ await this.processTimeSeriesNode({
5164
4180
  nodeName,
5165
4181
  accountId,
5166
4182
  fields
5167
4183
  });
5168
4184
  } else {
5169
- this.processCatalogNode({
4185
+ await this.processCatalogNode({
5170
4186
  nodeName,
5171
4187
  accountId,
5172
4188
  fields
@@ -5181,7 +4197,7 @@ OPTIONS(description="${this.description}")`;
5181
4197
  * @param {Array<string>} options.fields - Array of fields to fetch
5182
4198
  * @param {Object} options.storage - Storage instance
5183
4199
  */
5184
- processTimeSeriesNode({ nodeName, accountId, fields }) {
4200
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
5185
4201
  var _a;
5186
4202
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
5187
4203
  if (daysToFetch <= 0) {
@@ -5191,12 +4207,13 @@ OPTIONS(description="${this.description}")`;
5191
4207
  for (let i = 0; i < daysToFetch; i++) {
5192
4208
  const currentDate = new Date(startDate);
5193
4209
  currentDate.setDate(currentDate.getDate() + i);
5194
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
5195
- const data = this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
4210
+ const formattedDate = DateUtils2.formatDate(currentDate);
4211
+ const data = await this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
5196
4212
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
5197
4213
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
5198
4214
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
5199
- this.getStorageByNode(nodeName).saveData(preparedData);
4215
+ const storage = await this.getStorageByNode(nodeName);
4216
+ await storage.saveData(preparedData);
5200
4217
  }
5201
4218
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
5202
4219
  this.config.updateLastRequstedDate(currentDate);
@@ -5211,13 +4228,14 @@ OPTIONS(description="${this.description}")`;
5211
4228
  * @param {Array<string>} options.fields - Array of fields to fetch
5212
4229
  * @param {Object} options.storage - Storage instance
5213
4230
  */
5214
- processCatalogNode({ nodeName, accountId, fields }) {
4231
+ async processCatalogNode({ nodeName, accountId, fields }) {
5215
4232
  var _a;
5216
- const data = this.source.fetchData({ nodeName, accountId, fields });
4233
+ const data = await this.source.fetchData({ nodeName, accountId, fields });
5217
4234
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
5218
4235
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
5219
4236
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
5220
- this.getStorageByNode(nodeName).saveData(preparedData);
4237
+ const storage = await this.getStorageByNode(nodeName);
4238
+ await storage.saveData(preparedData);
5221
4239
  }
5222
4240
  }
5223
4241
  /**
@@ -5225,7 +4243,7 @@ OPTIONS(description="${this.description}")`;
5225
4243
  * @param {string} nodeName - Name of the node
5226
4244
  * @returns {Object} Storage instance
5227
4245
  */
5228
- getStorageByNode(nodeName) {
4246
+ async getStorageByNode(nodeName) {
5229
4247
  if (!("storages" in this)) {
5230
4248
  this.storages = {};
5231
4249
  }
@@ -5243,6 +4261,7 @@ OPTIONS(description="${this.description}")`;
5243
4261
  this.source.fieldsSchema[nodeName].fields,
5244
4262
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
5245
4263
  );
4264
+ await this.storages[nodeName].init();
5246
4265
  }
5247
4266
  return this.storages[nodeName];
5248
4267
  }
@@ -5259,7 +4278,7 @@ OPTIONS(description="${this.description}")`;
5259
4278
  };
5260
4279
  })();
5261
4280
  const TikTokAds = (function() {
5262
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
4281
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
5263
4282
  class TiktokMarketingApiProvider {
5264
4283
  constructor(appId, accessToken, appSecret, isSandbox = false) {
5265
4284
  this.BASE_URL = "https://business-api.tiktok.com/open_api/";
@@ -5283,7 +4302,7 @@ OPTIONS(description="${this.description}")`;
5283
4302
  getApiVersion() {
5284
4303
  return this.API_VERSION;
5285
4304
  }
5286
- makeRequest(options) {
4305
+ async makeRequest(options) {
5287
4306
  const { url, method, data } = options;
5288
4307
  const headers = {
5289
4308
  "Access-Token": this.accessToken,
@@ -5292,21 +4311,21 @@ OPTIONS(description="${this.description}")`;
5292
4311
  let backoff = this.INITIAL_BACKOFF;
5293
4312
  for (let retries = 0; retries < this.MAX_RETRIES; retries++) {
5294
4313
  try {
5295
- const response = EnvironmentAdapter2.fetch(url, {
4314
+ const response = await HttpUtils2.fetch(url, {
5296
4315
  method,
5297
4316
  headers,
5298
- body: data ? JSON.stringify(data) : null,
5299
- muteHttpExceptions: true
4317
+ body: data ? JSON.stringify(data) : null
5300
4318
  });
5301
4319
  const responseCode = response.getResponseCode();
4320
+ const text = await response.getContentText();
5302
4321
  if (responseCode !== this.SUCCESS_RESPONSE_CODE) {
5303
- throw new Error(`TikTok API error: ${response.getContentText()}`);
4322
+ throw new Error(`TikTok API error: ${text}`);
5304
4323
  }
5305
- const jsonData = JSON.parse(response.getContentText());
4324
+ const jsonData = JSON.parse(text);
5306
4325
  if (jsonData.code !== this.SUCCESS_CODE) {
5307
4326
  if (jsonData.code === this.RATE_LIMIT_CODE) {
5308
4327
  console.error("TikTok Marketing API rate limit exceeded. Retrying...");
5309
- EnvironmentAdapter2.sleep(backoff);
4328
+ await AsyncUtils2.delay(backoff);
5310
4329
  backoff *= 2;
5311
4330
  continue;
5312
4331
  }
@@ -5315,7 +4334,7 @@ OPTIONS(description="${this.description}")`;
5315
4334
  return jsonData;
5316
4335
  } catch (error) {
5317
4336
  if (retries < this.MAX_RETRIES - 1 && error.message.includes("rate limit")) {
5318
- EnvironmentAdapter2.sleep(backoff);
4337
+ await AsyncUtils2.delay(backoff);
5319
4338
  backoff *= 2;
5320
4339
  } else {
5321
4340
  throw error;
@@ -5323,7 +4342,7 @@ OPTIONS(description="${this.description}")`;
5323
4342
  }
5324
4343
  }
5325
4344
  }
5326
- handlePagination(endpoint, params = {}) {
4345
+ async handlePagination(endpoint, params = {}) {
5327
4346
  let allData = [];
5328
4347
  let page = 1;
5329
4348
  let hasMorePages = true;
@@ -5331,7 +4350,7 @@ OPTIONS(description="${this.description}")`;
5331
4350
  while (hasMorePages) {
5332
4351
  const paginatedParams = { ...params, page, page_size: pageSize };
5333
4352
  const url = this.buildUrl(endpoint, paginatedParams);
5334
- const response = this.makeRequest({ url, method: "GET" });
4353
+ const response = await this.makeRequest({ url, method: "GET" });
5335
4354
  const pageData = response.data.list || [];
5336
4355
  allData = allData.concat(pageData);
5337
4356
  const total = response.data.page_info ? response.data.page_info.total_number : 0;
@@ -5339,7 +4358,7 @@ OPTIONS(description="${this.description}")`;
5339
4358
  hasMorePages = currentCount < total && pageData.length > 0;
5340
4359
  page++;
5341
4360
  if (hasMorePages) {
5342
- EnvironmentAdapter2.sleep(100);
4361
+ await AsyncUtils2.delay(100);
5343
4362
  }
5344
4363
  }
5345
4364
  return allData;
@@ -6422,7 +5441,7 @@ OPTIONS(description="${this.description}")`;
6422
5441
  * @param {Date} endDate - End date for time-series data (optional)
6423
5442
  * @return {array} - Array of data objects
6424
5443
  */
6425
- fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
5444
+ async fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
6426
5445
  if (!this.fieldsSchema[nodeName]) {
6427
5446
  throw new Error(`Unknown node type: ${nodeName}`);
6428
5447
  }
@@ -6443,8 +5462,8 @@ OPTIONS(description="${this.description}")`;
6443
5462
  let formattedStartDate = null;
6444
5463
  let formattedEndDate = null;
6445
5464
  if (startDate) {
6446
- formattedStartDate = EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd");
6447
- formattedEndDate = endDate ? EnvironmentAdapter2.formatDate(endDate, "UTC", "yyyy-MM-dd") : formattedStartDate;
5465
+ formattedStartDate = DateUtils2.formatDate(startDate);
5466
+ formattedEndDate = endDate ? DateUtils2.formatDate(endDate) : formattedStartDate;
6448
5467
  }
6449
5468
  let filtering = null;
6450
5469
  if (this.config.IncludeDeleted && this.config.IncludeDeleted.value) {
@@ -6666,11 +5685,11 @@ OPTIONS(description="${this.description}")`;
6666
5685
  }
6667
5686
  };
6668
5687
  var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector2 {
6669
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
5688
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
6670
5689
  super(config, source, null, runConfig);
6671
5690
  this.storageName = storageName;
6672
5691
  }
6673
- startImportProcess() {
5692
+ async startImportProcess() {
6674
5693
  try {
6675
5694
  let advertiserIds = TikTokAdsHelper.parseAdvertiserIds(this.config.AdvertiserIDs.value || "");
6676
5695
  if (!advertiserIds || advertiserIds.length === 0) {
@@ -6694,7 +5713,7 @@ OPTIONS(description="${this.description}")`;
6694
5713
  }
6695
5714
  }
6696
5715
  if (Object.keys(catalogNodes).length > 0) {
6697
- this.importCatalogData(catalogNodes, advertiserIds);
5716
+ await this.importCatalogData(catalogNodes, advertiserIds);
6698
5717
  }
6699
5718
  if (Object.keys(timeSeriesNodes).length > 0) {
6700
5719
  try {
@@ -6703,7 +5722,7 @@ OPTIONS(description="${this.description}")`;
6703
5722
  this.config.logMessage("There is nothing to import in this data range");
6704
5723
  return;
6705
5724
  }
6706
- this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
5725
+ await this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
6707
5726
  } catch (error) {
6708
5727
  this.config.logMessage(`Error determining date range: ${error.message}`);
6709
5728
  console.error(error.stack);
@@ -6722,35 +5741,36 @@ OPTIONS(description="${this.description}")`;
6722
5741
  }
6723
5742
  /**
6724
5743
  * Imports all catalog (non-time-series) data types
6725
- *
5744
+ *
6726
5745
  * @param {object} catalogNodes - Object with node names as keys and field arrays as values
6727
5746
  * @param {array} advertiserIds - List of advertiser IDs to fetch data for
6728
5747
  */
6729
- importCatalogData(catalogNodes, advertiserIds) {
5748
+ async importCatalogData(catalogNodes, advertiserIds) {
6730
5749
  for (var nodeName in catalogNodes) {
6731
5750
  this.config.logMessage(`Starting import for ${nodeName} data...`);
6732
- this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
5751
+ await this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
6733
5752
  }
6734
5753
  }
6735
5754
  /**
6736
5755
  * Imports catalog (not time series) data
6737
- *
5756
+ *
6738
5757
  * @param {string} nodeName - Node name
6739
5758
  * @param {array} advertiserIds - List of advertiser IDs
6740
5759
  * @param {array} fields - List of fields
6741
5760
  */
6742
- startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
5761
+ async startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
6743
5762
  var _a;
6744
5763
  this.config.logMessage(`Fetching all available fields for ${nodeName}`);
6745
5764
  for (var i in advertiserIds) {
6746
5765
  let advertiserId = advertiserIds[i];
6747
5766
  try {
6748
- let data = this.source.fetchData(nodeName, advertiserId, fields);
5767
+ let data = await this.source.fetchData(nodeName, advertiserId, fields);
6749
5768
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for advertiser ${advertiserId}` : `No records have been fetched`);
6750
5769
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
6751
5770
  try {
6752
5771
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
6753
- this.getStorageByNode(nodeName).saveData(preparedData);
5772
+ const storage = await this.getStorageByNode(nodeName);
5773
+ await storage.saveData(preparedData);
6754
5774
  } catch (storageError) {
6755
5775
  this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
6756
5776
  console.error(`Error details: ${storageError.stack}`);
@@ -6764,29 +5784,30 @@ OPTIONS(description="${this.description}")`;
6764
5784
  }
6765
5785
  /**
6766
5786
  * Imports time series data
6767
- *
5787
+ *
6768
5788
  * @param {array} advertiserIds - List of advertiser IDs
6769
5789
  * @param {object} timeSeriesNodes - Object of properties, each is array of fields
6770
5790
  * @param {Date} startDate - Start date
6771
5791
  * @param {number} daysToFetch - Number of days to fetch
6772
5792
  */
6773
- startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
5793
+ async startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
6774
5794
  var _a;
6775
5795
  for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
6776
5796
  const currentDate = new Date(startDate);
6777
5797
  currentDate.setDate(currentDate.getDate() + daysShift);
6778
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
5798
+ const formattedDate = DateUtils2.formatDate(currentDate);
6779
5799
  this.config.logMessage(`Processing data for date: ${formattedDate}`);
6780
5800
  for (let advertiserId of advertiserIds) {
6781
5801
  for (var nodeName in timeSeriesNodes) {
6782
5802
  try {
6783
5803
  this.config.logMessage(`Start importing data for ${formattedDate}: ${advertiserId}/${nodeName}`);
6784
- let data = this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
5804
+ let data = await this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
6785
5805
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
6786
5806
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
6787
5807
  try {
6788
5808
  const preparedData = data.length ? this.addMissingFieldsToData(data, timeSeriesNodes[nodeName]) : data;
6789
- this.getStorageByNode(nodeName).saveData(preparedData);
5809
+ const storage = await this.getStorageByNode(nodeName);
5810
+ await storage.saveData(preparedData);
6790
5811
  } catch (storageError) {
6791
5812
  this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
6792
5813
  console.error(`Error details: ${storageError.stack}`);
@@ -6810,7 +5831,7 @@ OPTIONS(description="${this.description}")`;
6810
5831
  * @param {array} requestedFields - List of requested fields
6811
5832
  * @return {AbstractStorage} - Storage instance
6812
5833
  */
6813
- getStorageByNode(nodeName) {
5834
+ async getStorageByNode(nodeName) {
6814
5835
  if (!("storages" in this)) {
6815
5836
  this.storages = {};
6816
5837
  }
@@ -6832,6 +5853,7 @@ OPTIONS(description="${this.description}")`;
6832
5853
  this.source.fieldsSchema[nodeName]["fields"] || {},
6833
5854
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
6834
5855
  );
5856
+ await this.storages[nodeName].init();
6835
5857
  }
6836
5858
  return this.storages[nodeName];
6837
5859
  }
@@ -6849,35 +5871,9 @@ OPTIONS(description="${this.description}")`;
6849
5871
  this.config.logMessage(`Cleaning up data older than ${keepDays} days...`);
6850
5872
  const cutoffDate = /* @__PURE__ */ new Date();
6851
5873
  cutoffDate.setDate(cutoffDate.getDate() - keepDays);
6852
- const formattedCutoffDate = EnvironmentAdapter2.formatDate(cutoffDate, "UTC", "yyyy-MM-dd");
5874
+ DateUtils2.formatDate(cutoffDate);
6853
5875
  for (var nodeName in this.source.fieldsSchema) {
6854
- if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) {
6855
- try {
6856
- const storage = this.getStorageByNode(nodeName);
6857
- if (storage instanceof GoogleSheetsStorage) {
6858
- const dateField = this.source.fieldsSchema[nodeName]["fields"]["date_start"] ? "date_start" : "stat_time_day";
6859
- let keysToDelete = [];
6860
- for (const uniqueKey in storage.values) {
6861
- const record = storage.values[uniqueKey];
6862
- const rowDate = record[dateField];
6863
- if (rowDate && rowDate < cutoffDate) {
6864
- keysToDelete.push(uniqueKey);
6865
- }
6866
- }
6867
- let deletedCount = 0;
6868
- for (const key of keysToDelete) {
6869
- storage.deleteRecord(key);
6870
- deletedCount++;
6871
- }
6872
- if (deletedCount > 0) {
6873
- this.config.logMessage(`Deleted ${deletedCount} rows from ${nodeName} that were older than ${formattedCutoffDate}`);
6874
- }
6875
- }
6876
- } catch (error) {
6877
- this.config.logMessage(`Error cleaning up old data from ${nodeName}: ${error.message}`);
6878
- console.error(`Error details: ${error.stack}`);
6879
- }
6880
- }
5876
+ if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) ;
6881
5877
  }
6882
5878
  }
6883
5879
  };
@@ -6894,7 +5890,7 @@ OPTIONS(description="${this.description}")`;
6894
5890
  };
6895
5891
  })();
6896
5892
  const RedditAds = (function() {
6897
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
5893
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
6898
5894
  const RedditAdsHelper = {
6899
5895
  /**
6900
5896
  * Parse fields string into a structured object
@@ -9930,7 +8926,7 @@ OPTIONS(description="${this.description}")`;
9930
8926
  * @param {Date|null} startDate - The start date for data fetching.
9931
8927
  * @returns {Array} An array of data records.
9932
8928
  */
9933
- fetchData(nodeName, accountId, fields, startDate = null) {
8929
+ async fetchData(nodeName, accountId, fields, startDate = null) {
9934
8930
  var _a;
9935
8931
  console.log(`Fetching data from ${nodeName}/${accountId} for ${startDate}`);
9936
8932
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -9938,7 +8934,7 @@ OPTIONS(description="${this.description}")`;
9938
8934
  if (missingKeys.length > 0) {
9939
8935
  throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
9940
8936
  }
9941
- const tokenResponse = this.getRedditAccessToken(
8937
+ const tokenResponse = await this.getRedditAccessToken(
9942
8938
  this.config.ClientId.value,
9943
8939
  this.config.ClientSecret.value,
9944
8940
  this.config.RedirectUri.value,
@@ -9948,7 +8944,7 @@ OPTIONS(description="${this.description}")`;
9948
8944
  this.config.AccessToken.value = tokenResponse.accessToken;
9949
8945
  }
9950
8946
  const baseUrl = "https://ads-api.reddit.com/api/v3/";
9951
- let formattedDate = startDate ? EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd") : null;
8947
+ let formattedDate = startDate ? DateUtils2.formatDate(startDate) : null;
9952
8948
  let headers = {
9953
8949
  "Accept": "application/json",
9954
8950
  "User-Agent": this.config.UserAgent.value,
@@ -9974,8 +8970,9 @@ OPTIONS(description="${this.description}")`;
9974
8970
  let nextPageURL = finalUrl;
9975
8971
  while (nextPageURL) {
9976
8972
  try {
9977
- const response = this.urlFetchWithRetry(nextPageURL, options);
9978
- const jsonData = JSON.parse(response.getContentText());
8973
+ const response = await this.urlFetchWithRetry(nextPageURL, options);
8974
+ const text = await response.getContentText();
8975
+ const jsonData = JSON.parse(text);
9979
8976
  if ("data" in jsonData) {
9980
8977
  nextPageURL = jsonData.pagination ? jsonData.pagination.next_url : null;
9981
8978
  if (jsonData && jsonData.data && jsonData.data.metrics) {
@@ -9992,7 +8989,7 @@ OPTIONS(description="${this.description}")`;
9992
8989
  }
9993
8990
  } catch (error) {
9994
8991
  if (error.statusCode === HTTP_STATUS2.UNAUTHORIZED) {
9995
- const newTokenResponse = this.getRedditAccessToken(
8992
+ const newTokenResponse = await this.getRedditAccessToken(
9996
8993
  this.config.ClientId.value,
9997
8994
  this.config.ClientSecret.value,
9998
8995
  this.config.RedirectUri.value,
@@ -10019,12 +9016,12 @@ OPTIONS(description="${this.description}")`;
10019
9016
  * @param {string} refreshToken - The refresh token.
10020
9017
  * @returns {Object} An object with a success flag and either the access token or an error message.
10021
9018
  */
10022
- getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
9019
+ async getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
10023
9020
  const url = "https://www.reddit.com/api/v1/access_token";
10024
9021
  const headers = {
10025
9022
  "Content-Type": "application/x-www-form-urlencoded",
10026
9023
  "User-Agent": this.config.UserAgent.value,
10027
- "Authorization": "Basic " + EnvironmentAdapter2.base64Encode(clientId + ":" + clientSecret)
9024
+ "Authorization": "Basic " + CryptoUtils2.base64Encode(clientId + ":" + clientSecret)
10028
9025
  };
10029
9026
  const payload = {
10030
9027
  "grant_type": "refresh_token",
@@ -10040,8 +9037,8 @@ OPTIONS(description="${this.description}")`;
10040
9037
  muteHttpExceptions: true
10041
9038
  };
10042
9039
  try {
10043
- const response = EnvironmentAdapter2.fetch(url, options);
10044
- const result = response.getContentText();
9040
+ const response = await HttpUtils2.fetch(url, options);
9041
+ const result = await response.getContentText();
10045
9042
  const json = JSON.parse(result);
10046
9043
  if (json.error) {
10047
9044
  return { success: false, message: json.error };
@@ -10426,7 +9423,7 @@ OPTIONS(description="${this.description}")`;
10426
9423
  }
10427
9424
  };
10428
9425
  var RedditAdsConnector = class RedditAdsConnector extends AbstractConnector2 {
10429
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9426
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10430
9427
  super(config, source, null, runConfig);
10431
9428
  this.storageName = storageName;
10432
9429
  }
@@ -10434,12 +9431,12 @@ OPTIONS(description="${this.description}")`;
10434
9431
  * Main method - entry point for the import process
10435
9432
  * Processes all nodes defined in the fields configuration
10436
9433
  */
10437
- startImportProcess() {
9434
+ async startImportProcess() {
10438
9435
  const fields = RedditAdsHelper.parseFields(this.config.Fields.value);
10439
9436
  const accountIds = RedditAdsHelper.parseAccountIds(this.config.AccountIDs.value);
10440
9437
  for (const accountId of accountIds) {
10441
9438
  for (const nodeName in fields) {
10442
- this.processNode({
9439
+ await this.processNode({
10443
9440
  nodeName,
10444
9441
  accountId,
10445
9442
  fields: fields[nodeName] || []
@@ -10454,15 +9451,15 @@ OPTIONS(description="${this.description}")`;
10454
9451
  * @param {string} options.accountId - Account ID
10455
9452
  * @param {Array<string>} options.fields - Array of fields to fetch
10456
9453
  */
10457
- processNode({ nodeName, accountId, fields }) {
9454
+ async processNode({ nodeName, accountId, fields }) {
10458
9455
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
10459
- this.processTimeSeriesNode({
9456
+ await this.processTimeSeriesNode({
10460
9457
  nodeName,
10461
9458
  accountId,
10462
9459
  fields
10463
9460
  });
10464
9461
  } else {
10465
- this.processCatalogNode({
9462
+ await this.processCatalogNode({
10466
9463
  nodeName,
10467
9464
  accountId,
10468
9465
  fields
@@ -10477,7 +9474,7 @@ OPTIONS(description="${this.description}")`;
10477
9474
  * @param {Array<string>} options.fields - Array of fields to fetch
10478
9475
  * @param {Object} options.storage - Storage instance
10479
9476
  */
10480
- processTimeSeriesNode({ nodeName, accountId, fields }) {
9477
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
10481
9478
  var _a;
10482
9479
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
10483
9480
  if (daysToFetch <= 0) {
@@ -10487,13 +9484,14 @@ OPTIONS(description="${this.description}")`;
10487
9484
  for (let i = 0; i < daysToFetch; i++) {
10488
9485
  const currentDate = new Date(startDate);
10489
9486
  currentDate.setDate(currentDate.getDate() + i);
10490
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
9487
+ const formattedDate = DateUtils2.formatDate(currentDate);
10491
9488
  this.config.logMessage(`Start importing data for ${formattedDate}: ${accountId}/${nodeName}`);
10492
- const data = this.source.fetchData(nodeName, accountId, fields, currentDate);
9489
+ const data = await this.source.fetchData(nodeName, accountId, fields, currentDate);
10493
9490
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
10494
9491
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10495
9492
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10496
- this.getStorageByNode(nodeName).saveData(preparedData);
9493
+ const storage = await this.getStorageByNode(nodeName);
9494
+ await storage.saveData(preparedData);
10497
9495
  }
10498
9496
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
10499
9497
  this.config.updateLastRequstedDate(currentDate);
@@ -10508,13 +9506,14 @@ OPTIONS(description="${this.description}")`;
10508
9506
  * @param {Array<string>} options.fields - Array of fields to fetch
10509
9507
  * @param {Object} options.storage - Storage instance
10510
9508
  */
10511
- processCatalogNode({ nodeName, accountId, fields }) {
9509
+ async processCatalogNode({ nodeName, accountId, fields }) {
10512
9510
  var _a;
10513
- const data = this.source.fetchData(nodeName, accountId, fields);
9511
+ const data = await this.source.fetchData(nodeName, accountId, fields);
10514
9512
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for account ${accountId}` : `No records have been fetched`);
10515
9513
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10516
9514
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10517
- this.getStorageByNode(nodeName).saveData(preparedData);
9515
+ const storage = await this.getStorageByNode(nodeName);
9516
+ await storage.saveData(preparedData);
10518
9517
  }
10519
9518
  }
10520
9519
  /**
@@ -10522,7 +9521,7 @@ OPTIONS(description="${this.description}")`;
10522
9521
  * @param {string} nodeName - Name of the node
10523
9522
  * @returns {Object} Storage instance
10524
9523
  */
10525
- getStorageByNode(nodeName) {
9524
+ async getStorageByNode(nodeName) {
10526
9525
  if (!("storages" in this)) {
10527
9526
  this.storages = {};
10528
9527
  }
@@ -10540,6 +9539,7 @@ OPTIONS(description="${this.description}")`;
10540
9539
  this.source.fieldsSchema[nodeName].fields,
10541
9540
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
10542
9541
  );
9542
+ await this.storages[nodeName].init();
10543
9543
  }
10544
9544
  return this.storages[nodeName];
10545
9545
  }
@@ -10556,7 +9556,7 @@ OPTIONS(description="${this.description}")`;
10556
9556
  };
10557
9557
  })();
10558
9558
  const OpenHolidays = (function() {
10559
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
9559
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10560
9560
  var publicHolidaysFields = {
10561
9561
  id: {
10562
9562
  type: "string",
@@ -10664,10 +9664,10 @@ OPTIONS(description="${this.description}")`;
10664
9664
  * @param {string} [opts.end_time]
10665
9665
  * @returns {Array<Object>}
10666
9666
  */
10667
- fetchData({ nodeName, fields = [], start_time, end_time }) {
9667
+ async fetchData({ nodeName, fields = [], start_time, end_time }) {
10668
9668
  switch (nodeName) {
10669
9669
  case "publicHolidays":
10670
- return this._fetchPublicHolidays({ fields, start_time, end_time });
9670
+ return await this._fetchPublicHolidays({ fields, start_time, end_time });
10671
9671
  default:
10672
9672
  throw new Error(`Unknown node: ${nodeName}`);
10673
9673
  }
@@ -10680,10 +9680,10 @@ OPTIONS(description="${this.description}")`;
10680
9680
  * @param {string} options.end_time - End date for data fetch (YYYY-MM-DD format)
10681
9681
  * @returns {Array} Array of holiday data
10682
9682
  */
10683
- _fetchPublicHolidays({ fields, start_time, end_time }) {
9683
+ async _fetchPublicHolidays({ fields, start_time, end_time }) {
10684
9684
  let countryIsoCode = this.config.countryIsoCode.value;
10685
9685
  let languageIsoCode = this.config.languageIsoCode.value;
10686
- const holidays = this.makeRequest({
9686
+ const holidays = await this.makeRequest({
10687
9687
  endpoint: `PublicHolidays?countryIsoCode=${countryIsoCode}&languageIsoCode=${languageIsoCode}&validFrom=${start_time}&validTo=${end_time}`
10688
9688
  });
10689
9689
  if (!holidays || !holidays.length) {
@@ -10710,12 +9710,13 @@ OPTIONS(description="${this.description}")`;
10710
9710
  * @param {string} options.endpoint - API endpoint path (e.g., "PublicHolidays?countryIsoCode=CH&...")
10711
9711
  * @returns {Object} - API response parsed from JSON
10712
9712
  */
10713
- makeRequest({ endpoint }) {
9713
+ async makeRequest({ endpoint }) {
10714
9714
  const baseUrl = "https://openholidaysapi.org/";
10715
9715
  const url = `${baseUrl}${endpoint}`;
10716
9716
  console.log(`OpenHolidays API Request URL:`, url);
10717
- const response = EnvironmentAdapter2.fetch(url, { "method": "get", "muteHttpExceptions": true });
10718
- const result = JSON.parse(response.getContentText());
9717
+ const response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
9718
+ const text = await response.getContentText();
9719
+ const result = JSON.parse(text);
10719
9720
  return result;
10720
9721
  }
10721
9722
  /**
@@ -10741,7 +9742,7 @@ OPTIONS(description="${this.description}")`;
10741
9742
  }
10742
9743
  };
10743
9744
  var OpenHolidaysConnector = class OpenHolidaysConnector extends AbstractConnector2 {
10744
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9745
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10745
9746
  super(config, source, null, runConfig);
10746
9747
  this.storageName = storageName;
10747
9748
  }
@@ -10749,10 +9750,10 @@ OPTIONS(description="${this.description}")`;
10749
9750
  * Main method - entry point for the import process
10750
9751
  * Processes all nodes defined in the fields configuration
10751
9752
  */
10752
- startImportProcess() {
9753
+ async startImportProcess() {
10753
9754
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
10754
9755
  for (const nodeName in fields) {
10755
- this.processNode({
9756
+ await this.processNode({
10756
9757
  nodeName,
10757
9758
  fields: fields[nodeName] || []
10758
9759
  });
@@ -10764,12 +9765,12 @@ OPTIONS(description="${this.description}")`;
10764
9765
  * @param {string} options.nodeName - Name of the node to process
10765
9766
  * @param {Array<string>} options.fields - Array of fields to fetch
10766
9767
  */
10767
- processNode({ nodeName, fields }) {
9768
+ async processNode({ nodeName, fields }) {
10768
9769
  const storage = this.getStorageByNode(nodeName);
10769
9770
  if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
10770
- this.processTimeSeriesNode({ nodeName, fields, storage });
9771
+ await this.processTimeSeriesNode({ nodeName, fields, storage });
10771
9772
  } else {
10772
- this.processCatalogNode({ nodeName, fields, storage });
9773
+ await this.processCatalogNode({ nodeName, fields, storage });
10773
9774
  }
10774
9775
  }
10775
9776
  /**
@@ -10779,14 +9780,14 @@ OPTIONS(description="${this.description}")`;
10779
9780
  * @param {Array<string>} options.fields - Array of fields to fetch
10780
9781
  * @param {Object} options.storage - Storage instance
10781
9782
  */
10782
- processTimeSeriesNode({ nodeName, fields, storage }) {
9783
+ async processTimeSeriesNode({ nodeName, fields, storage }) {
10783
9784
  var _a;
10784
9785
  const dateRange = this.prepareDateRange();
10785
9786
  if (!dateRange) {
10786
9787
  console.log("No date range available for time series data");
10787
9788
  return;
10788
9789
  }
10789
- const data = this.source.fetchData({
9790
+ const data = await this.source.fetchData({
10790
9791
  nodeName,
10791
9792
  start_time: dateRange.startDate,
10792
9793
  end_time: dateRange.endDate,
@@ -10795,7 +9796,7 @@ OPTIONS(description="${this.description}")`;
10795
9796
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
10796
9797
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
10797
9798
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
10798
- storage.saveData(preparedData);
9799
+ await storage.saveData(preparedData);
10799
9800
  }
10800
9801
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
10801
9802
  this.config.updateLastRequstedDate(new Date(dateRange.endDate));
@@ -10808,7 +9809,7 @@ OPTIONS(description="${this.description}")`;
10808
9809
  * @param {Array<string>} options.fields - Array of fields to fetch
10809
9810
  * @param {Object} options.storage - Storage instance
10810
9811
  */
10811
- processCatalogNode({ nodeName, fields, storage }) {
9812
+ async processCatalogNode({ nodeName, fields, storage }) {
10812
9813
  console.log(`Catalog node processing not implemented for ${nodeName}`);
10813
9814
  }
10814
9815
  /**
@@ -10816,7 +9817,7 @@ OPTIONS(description="${this.description}")`;
10816
9817
  * @param {string} nodeName - Name of the node
10817
9818
  * @returns {Object} Storage instance
10818
9819
  */
10819
- getStorageByNode(nodeName) {
9820
+ async getStorageByNode(nodeName) {
10820
9821
  if (!("storages" in this)) {
10821
9822
  this.storages = {};
10822
9823
  }
@@ -10834,6 +9835,7 @@ OPTIONS(description="${this.description}")`;
10834
9835
  this.source.fieldsSchema[nodeName].fields,
10835
9836
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
10836
9837
  );
9838
+ await this.storages[nodeName].init();
10837
9839
  }
10838
9840
  return this.storages[nodeName];
10839
9841
  }
@@ -10849,8 +9851,8 @@ OPTIONS(description="${this.description}")`;
10849
9851
  const endDate = new Date(startDate);
10850
9852
  endDate.setDate(endDate.getDate() + daysToFetch - 1);
10851
9853
  return {
10852
- startDate: EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd"),
10853
- endDate: EnvironmentAdapter2.formatDate(endDate, "UTC", "yyyy-MM-dd")
9854
+ startDate: DateUtils2.formatDate(startDate),
9855
+ endDate: DateUtils2.formatDate(endDate)
10854
9856
  };
10855
9857
  }
10856
9858
  };
@@ -10866,7 +9868,7 @@ OPTIONS(description="${this.description}")`;
10866
9868
  };
10867
9869
  })();
10868
9870
  const OpenExchangeRates = (function() {
10869
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
9871
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10870
9872
  var historicalFields = {
10871
9873
  date: {
10872
9874
  type: "DATE",
@@ -10954,12 +9956,12 @@ OPTIONS(description="${this.description}")`;
10954
9956
  this.fieldsSchema = OpenExchangeRatesFieldsSchema;
10955
9957
  }
10956
9958
  /*
10957
- @param date The requested date as a Date object
9959
+ @param date The requested date as a Date object
10958
9960
 
10959
- @return data array
9961
+ @return data array
10960
9962
 
10961
- */
10962
- fetchData(date) {
9963
+ */
9964
+ async fetchData(date) {
10963
9965
  let data = [];
10964
9966
  let base = this.config.base.value;
10965
9967
  let app_id = this.config.AppId.value;
@@ -10967,13 +9969,14 @@ OPTIONS(description="${this.description}")`;
10967
9969
  if (this.config.Symbols.value) {
10968
9970
  symbols = "&symbols=" + String(this.config.Symbols.value).replace(/[^A-Z,]/g, "");
10969
9971
  }
10970
- var date = EnvironmentAdapter2.formatDate(date, "UTC", "yyyy-MM-dd");
9972
+ var date = DateUtils2.formatDate(date);
10971
9973
  const urlWithoutKey = `https://openexchangerates.org/api/historical/${date}.json?base=${base}${symbols}`;
10972
9974
  console.log(`OpenExchangeRates API URL:`, urlWithoutKey);
10973
9975
  const url = `${urlWithoutKey}&app_id=${app_id}`;
10974
9976
  this.config.logMessage(`Fetching rates for ${date}`);
10975
- var response = EnvironmentAdapter2.fetch(url, { "method": "get", "muteHttpExceptions": true });
10976
- var historical = JSON.parse(response.getContentText());
9977
+ var response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
9978
+ var text = await response.getContentText();
9979
+ var historical = JSON.parse(text);
10977
9980
  for (var currency in historical["rates"]) {
10978
9981
  data.push({
10979
9982
  date: new Date(date),
@@ -10986,7 +9989,7 @@ OPTIONS(description="${this.description}")`;
10986
9989
  }
10987
9990
  };
10988
9991
  var OpenExchangeRatesConnector = class OpenExchangeRatesConnector extends AbstractConnector2 {
10989
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
9992
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
10990
9993
  super(config, source, null, runConfig);
10991
9994
  this.storageName = storageName;
10992
9995
  }
@@ -10999,11 +10002,11 @@ OPTIONS(description="${this.description}")`;
10999
10002
  * Main method - entry point for the import process
11000
10003
  * Processes all nodes defined in the fields configuration
11001
10004
  */
11002
- startImportProcess() {
10005
+ async startImportProcess() {
11003
10006
  var _a;
11004
10007
  const fields = ConnectorUtils.parseFields((_a = this.config.Fields) == null ? void 0 : _a.value);
11005
10008
  for (const nodeName in fields) {
11006
- this.processNode({
10009
+ await this.processNode({
11007
10010
  nodeName,
11008
10011
  fields: fields[nodeName] || []
11009
10012
  });
@@ -11015,11 +10018,11 @@ OPTIONS(description="${this.description}")`;
11015
10018
  * @param {string} options.nodeName - Name of the node to process
11016
10019
  * @param {Array<string>} options.fields - Array of fields to fetch
11017
10020
  */
11018
- processNode({ nodeName, fields }) {
10021
+ async processNode({ nodeName, fields }) {
11019
10022
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
11020
- this.processTimeSeriesNode({ nodeName, fields });
10023
+ await this.processTimeSeriesNode({ nodeName, fields });
11021
10024
  } else {
11022
- this.processCatalogNode({ nodeName, fields });
10025
+ await this.processCatalogNode({ nodeName, fields });
11023
10026
  }
11024
10027
  }
11025
10028
  /**
@@ -11028,7 +10031,7 @@ OPTIONS(description="${this.description}")`;
11028
10031
  * @param {string} options.nodeName - Name of the node
11029
10032
  * @param {Array<string>} options.fields - Array of fields to fetch
11030
10033
  */
11031
- processTimeSeriesNode({ nodeName, fields }) {
10034
+ async processTimeSeriesNode({ nodeName, fields }) {
11032
10035
  var _a;
11033
10036
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
11034
10037
  if (daysToFetch <= 0) {
@@ -11038,11 +10041,12 @@ OPTIONS(description="${this.description}")`;
11038
10041
  for (let daysShift = 0; daysShift < daysToFetch; daysShift++) {
11039
10042
  const currentDate = new Date(startDate);
11040
10043
  currentDate.setDate(currentDate.getDate() + daysShift);
11041
- let data = this.source.fetchData(currentDate);
10044
+ let data = await this.source.fetchData(currentDate);
11042
10045
  this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
11043
10046
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
11044
10047
  const preparedData = data.length ? data : [];
11045
- this.getStorageByNode(nodeName).saveData(preparedData);
10048
+ const storage = await this.getStorageByNode(nodeName);
10049
+ await storage.saveData(preparedData);
11046
10050
  }
11047
10051
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
11048
10052
  this.config.updateLastRequstedDate(currentDate);
@@ -11055,7 +10059,7 @@ OPTIONS(description="${this.description}")`;
11055
10059
  * @param {string} options.nodeName - Name of the node
11056
10060
  * @param {Array<string>} options.fields - Array of fields to fetch
11057
10061
  */
11058
- processCatalogNode({ nodeName, fields }) {
10062
+ async processCatalogNode({ nodeName, fields }) {
11059
10063
  console.log(`Catalog node processing not implemented for ${nodeName}`);
11060
10064
  }
11061
10065
  //---- getStorageName -------------------------------------------------
@@ -11067,7 +10071,7 @@ OPTIONS(description="${this.description}")`;
11067
10071
  * @return AbstractStorage
11068
10072
  *
11069
10073
  */
11070
- getStorageByNode(nodeName) {
10074
+ async getStorageByNode(nodeName) {
11071
10075
  if (!("storages" in this)) {
11072
10076
  this.storages = {};
11073
10077
  }
@@ -11085,6 +10089,7 @@ OPTIONS(description="${this.description}")`;
11085
10089
  this.source.fieldsSchema[nodeName]["fields"],
11086
10090
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
11087
10091
  );
10092
+ await this.storages[nodeName].init();
11088
10093
  }
11089
10094
  return this.storages[nodeName];
11090
10095
  }
@@ -11101,7 +10106,7 @@ OPTIONS(description="${this.description}")`;
11101
10106
  };
11102
10107
  })();
11103
10108
  const MicrosoftAds = (function() {
11104
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
10109
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
11105
10110
  const MicrosoftAdsHelper = {
11106
10111
  /**
11107
10112
  * Parse fields string into a structured object
@@ -11124,7 +10129,7 @@ OPTIONS(description="${this.description}")`;
11124
10129
  * @param {number} [opts.interval=5000]
11125
10130
  * @returns {Object}
11126
10131
  */
11127
- pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
10132
+ async pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
11128
10133
  const startTime = Date.now();
11129
10134
  const timeout = 15 * 60 * 1e3;
11130
10135
  let statusResult;
@@ -11133,9 +10138,10 @@ OPTIONS(description="${this.description}")`;
11133
10138
  if (Date.now() - startTime > timeout) {
11134
10139
  throw new Error("Polling timed out after 15 minutes");
11135
10140
  }
11136
- EnvironmentAdapter2.sleep(interval);
11137
- const response = EnvironmentAdapter2.fetch(url, options);
11138
- statusResult = JSON.parse(response.getContentText());
10141
+ await AsyncUtils2.delay(interval);
10142
+ const response = await HttpUtils2.fetch(url, options);
10143
+ const text = await response.getContentText();
10144
+ statusResult = JSON.parse(text);
11139
10145
  } while (!isDone(statusResult));
11140
10146
  return statusResult;
11141
10147
  } catch (error) {
@@ -11151,13 +10157,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
11151
10157
  * @param {string} url
11152
10158
  * @returns {Array<Array<string>>}
11153
10159
  */
11154
- downloadCsvRows(url) {
11155
- const response = EnvironmentAdapter2.fetch(url);
11156
- const files = EnvironmentAdapter2.unzip(response.getBlob());
10160
+ async downloadCsvRows(url) {
10161
+ const response = await HttpUtils2.fetch(url);
10162
+ const blob = await response.getBlob();
10163
+ const files = FileUtils2.unzip(blob);
11157
10164
  const allRows = [];
11158
10165
  files.forEach((file) => {
11159
10166
  const csvText = file.getDataAsString();
11160
- const rows = EnvironmentAdapter2.parseCsv(csvText);
10167
+ const rows = FileUtils2.parseCsv(csvText);
11161
10168
  allRows.push(...rows);
11162
10169
  });
11163
10170
  return allRows;
@@ -12284,7 +11291,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12284
11291
  /**
12285
11292
  * Retrieve and store an OAuth access token using the refresh token
12286
11293
  */
12287
- getAccessToken() {
11294
+ async getAccessToken() {
12288
11295
  var _a, _b;
12289
11296
  const tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
12290
11297
  const scopes = [
@@ -12310,8 +11317,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12310
11317
  body: Object.entries(form).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
12311
11318
  // TODO: body is for Node.js; refactor to centralize JSON option creation
12312
11319
  };
12313
- const resp = EnvironmentAdapter2.fetch(tokenUrl, options);
12314
- const json = JSON.parse(resp.getContentText());
11320
+ const resp = await HttpUtils2.fetch(tokenUrl, options);
11321
+ const text = await resp.getContentText();
11322
+ const json = JSON.parse(text);
12315
11323
  if (json.error) {
12316
11324
  throw new Error(`Token error: ${json.error} - ${json.error_description}`);
12317
11325
  }
@@ -12337,7 +11345,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12337
11345
  * @param {Function} [opts.onBatchReady] - Optional callback for batch processing
12338
11346
  * @returns {Array<Object>}
12339
11347
  */
12340
- fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
11348
+ async fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
12341
11349
  const schema = this.fieldsSchema[nodeName];
12342
11350
  if (schema.uniqueKeys) {
12343
11351
  const missingKeys = schema.uniqueKeys.filter((key) => !fields.includes(key));
@@ -12347,12 +11355,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12347
11355
  }
12348
11356
  switch (nodeName) {
12349
11357
  case "campaigns":
12350
- this._fetchCampaignData({ accountId, fields, onBatchReady });
11358
+ await this._fetchCampaignData({ accountId, fields, onBatchReady });
12351
11359
  return [];
12352
11360
  case "ad_performance_report":
12353
- return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
11361
+ return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
12354
11362
  case "user_location_performance_report":
12355
- return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
11363
+ return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
12356
11364
  default:
12357
11365
  throw new Error(`Unknown node: ${nodeName}`);
12358
11366
  }
@@ -12366,14 +11374,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12366
11374
  * @returns {void}
12367
11375
  * @private
12368
11376
  */
12369
- _fetchCampaignData({ accountId, fields, onBatchReady }) {
12370
- this.getAccessToken();
11377
+ async _fetchCampaignData({ accountId, fields, onBatchReady }) {
11378
+ await this.getAccessToken();
12371
11379
  this.config.logMessage(`Fetching Campaigns, AssetGroups and AdGroups for account ${accountId}...`);
12372
11380
  const entityTypes = ["Campaigns", "AssetGroups", "AdGroups"];
12373
11381
  const allRecords = [];
12374
11382
  let campaignRecords = [];
12375
11383
  for (const entityType of entityTypes) {
12376
- const records = this._downloadEntity({
11384
+ const records = await this._downloadEntity({
12377
11385
  submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByAccountIds",
12378
11386
  submitOpts: {
12379
11387
  method: "post",
@@ -12411,21 +11419,21 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12411
11419
  }
12412
11420
  const filteredMainData = MicrosoftAdsHelper.filterByFields(allRecords, fields);
12413
11421
  if (filteredMainData.length > 0) {
12414
- onBatchReady(filteredMainData);
11422
+ await onBatchReady(filteredMainData);
12415
11423
  }
12416
11424
  this.config.logMessage(`Fetching Keywords for account ${accountId} (processing by campaigns to avoid size limits)...`);
12417
11425
  const campaignIds = MicrosoftAdsHelper.extractCampaignIds(campaignRecords);
12418
11426
  this.config.logMessage(`Found ${campaignIds.length} campaigns, fetching Keywords in batches`);
12419
11427
  this.config.logMessage(`Campaign IDs: ${campaignIds.slice(0, 10).join(", ")}${campaignIds.length > 10 ? "..." : ""}`);
12420
11428
  let totalFetched = 0;
12421
- this._fetchEntityByCampaigns({
11429
+ await this._fetchEntityByCampaigns({
12422
11430
  accountId,
12423
11431
  entityType: "Keywords",
12424
11432
  campaignIds,
12425
- onBatchReady: (batchRecords) => {
11433
+ onBatchReady: async (batchRecords) => {
12426
11434
  totalFetched += batchRecords.length;
12427
11435
  const filteredBatch = MicrosoftAdsHelper.filterByFields(batchRecords, fields);
12428
- onBatchReady(filteredBatch);
11436
+ await onBatchReady(filteredBatch);
12429
11437
  }
12430
11438
  });
12431
11439
  this.config.logMessage(`${totalFetched} rows of Keywords were fetched for account ${accountId}`);
@@ -12439,15 +11447,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12439
11447
  * @returns {Array<Object>}
12440
11448
  * @private
12441
11449
  */
12442
- _downloadEntity({ submitUrl, submitOpts }) {
12443
- const submitResp = EnvironmentAdapter2.fetch(submitUrl, submitOpts);
12444
- const requestId = JSON.parse(submitResp.getContentText()).DownloadRequestId;
11450
+ async _downloadEntity({ submitUrl, submitOpts }) {
11451
+ const submitResp = await HttpUtils2.fetch(submitUrl, submitOpts);
11452
+ const text = await submitResp.getContentText();
11453
+ const requestId = JSON.parse(text).DownloadRequestId;
12445
11454
  const pollUrl = "https://bulk.api.bingads.microsoft.com/Bulk/v13/BulkDownloadStatus/Query";
12446
11455
  const pollOpts = Object.assign({}, submitOpts, {
12447
11456
  payload: JSON.stringify({ RequestId: requestId }),
12448
11457
  body: JSON.stringify({ RequestId: requestId })
12449
11458
  });
12450
- const pollResult = MicrosoftAdsHelper.pollUntilStatus({
11459
+ const pollResult = await MicrosoftAdsHelper.pollUntilStatus({
12451
11460
  url: pollUrl,
12452
11461
  options: pollOpts,
12453
11462
  isDone: (status) => {
@@ -12457,7 +11466,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12457
11466
  return status.RequestStatus === "Completed";
12458
11467
  }
12459
11468
  });
12460
- const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
11469
+ const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
12461
11470
  const result = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
12462
11471
  return result;
12463
11472
  }
@@ -12471,7 +11480,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12471
11480
  * @returns {Array<Object>} - Returns empty array if onBatchReady callback is provided, otherwise returns all records
12472
11481
  * @private
12473
11482
  */
12474
- _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
11483
+ async _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
12475
11484
  if (campaignIds.length === 0) {
12476
11485
  this.config.logMessage(`No active campaigns found for account ${accountId}, skipping ${entityType} fetch`);
12477
11486
  return [];
@@ -12481,9 +11490,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12481
11490
  const campaignBatch = campaignIds.slice(i, i + batchSize);
12482
11491
  this.config.logMessage(`Fetching ${entityType} for campaigns batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(campaignIds.length / batchSize)} (${campaignBatch.length} campaigns)`);
12483
11492
  try {
12484
- const batchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch });
11493
+ const batchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch });
12485
11494
  this.config.logMessage(`Fetched ${batchRecords.length} ${entityType.toLowerCase()} from current batch`);
12486
- onBatchReady(batchRecords);
11495
+ await onBatchReady(batchRecords);
12487
11496
  } catch (error) {
12488
11497
  if (error.message && error.message.includes("100MB")) {
12489
11498
  const newBatchSize = Math.max(1, Math.floor(batchSize / 2));
@@ -12491,9 +11500,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12491
11500
  for (let j = i; j < Math.min(i + batchSize, campaignIds.length); j += newBatchSize) {
12492
11501
  const smallerBatch = campaignIds.slice(j, j + newBatchSize);
12493
11502
  try {
12494
- const smallerBatchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
11503
+ const smallerBatchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
12495
11504
  this.config.logMessage(`Fetched ${smallerBatchRecords.length} ${entityType.toLowerCase()} from smaller batch (${smallerBatch.length} campaigns)`);
12496
- onBatchReady(smallerBatchRecords);
11505
+ await onBatchReady(smallerBatchRecords);
12497
11506
  } catch (smallerError) {
12498
11507
  if (smallerError.message && smallerError.message.includes("100MB")) {
12499
11508
  throw new Error(`Failed to fetch ${entityType}: batch size of ${smallerBatch.length} campaigns still exceeds 100MB limit`);
@@ -12520,7 +11529,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12520
11529
  * @returns {Array<Object>}
12521
11530
  * @private
12522
11531
  */
12523
- _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
11532
+ async _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
12524
11533
  const downloadBody = {
12525
11534
  Campaigns: campaignBatch.map((id) => ({
12526
11535
  CampaignId: Number(id),
@@ -12545,7 +11554,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12545
11554
  payload: JSON.stringify(downloadBody),
12546
11555
  body: JSON.stringify(downloadBody)
12547
11556
  };
12548
- return this._downloadEntity({
11557
+ return await this._downloadEntity({
12549
11558
  submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByCampaignIds",
12550
11559
  submitOpts
12551
11560
  });
@@ -12561,22 +11570,22 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12561
11570
  * @returns {Array<Object>}
12562
11571
  * @private
12563
11572
  */
12564
- _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
12565
- this.getAccessToken();
11573
+ async _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
11574
+ await this.getAccessToken();
12566
11575
  const schema = this.fieldsSchema[nodeName];
12567
- const submitResponse = this._submitReportRequest({
11576
+ const submitResponse = await this._submitReportRequest({
12568
11577
  accountId,
12569
11578
  fields,
12570
11579
  start_time,
12571
11580
  end_time,
12572
11581
  schema
12573
11582
  });
12574
- const pollResult = this._pollReportStatus({ submitResponse });
11583
+ const pollResult = await this._pollReportStatus({ submitResponse });
12575
11584
  if (!pollResult.ReportRequestStatus.ReportDownloadUrl) {
12576
11585
  this.config.logMessage(`No data available for the specified time period (${start_time} to ${end_time}). Report status: ${JSON.stringify(pollResult.ReportRequestStatus)}`);
12577
11586
  return [];
12578
11587
  }
12579
- const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
11588
+ const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
12580
11589
  const records = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
12581
11590
  return MicrosoftAdsHelper.filterByFields(records, fields);
12582
11591
  }
@@ -12591,7 +11600,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12591
11600
  * @returns {Object} - Submit response
12592
11601
  * @private
12593
11602
  */
12594
- _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
11603
+ async _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
12595
11604
  const dateRange = {
12596
11605
  CustomDateRangeStart: { Day: new Date(start_time).getDate(), Month: new Date(start_time).getMonth() + 1, Year: new Date(start_time).getFullYear() },
12597
11606
  CustomDateRangeEnd: { Day: new Date(end_time).getDate(), Month: new Date(end_time).getMonth() + 1, Year: new Date(end_time).getFullYear() },
@@ -12624,8 +11633,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12624
11633
  body: JSON.stringify({ ReportRequest: requestBody })
12625
11634
  // TODO: body is for Node.js; refactor to centralize JSON option creation
12626
11635
  };
12627
- const submitResp = EnvironmentAdapter2.fetch(submitUrl, submitOpts);
12628
- const submitResponseText = submitResp.getContentText();
11636
+ const submitResp = await HttpUtils2.fetch(submitUrl, submitOpts);
11637
+ const submitResponseText = await submitResp.getContentText();
12629
11638
  try {
12630
11639
  const submitResponse = JSON.parse(submitResponseText);
12631
11640
  if (submitResponse.OperationErrors && submitResponse.OperationErrors.length > 0) {
@@ -12649,7 +11658,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12649
11658
  * @returns {Object} - Poll result with report status
12650
11659
  * @private
12651
11660
  */
12652
- _pollReportStatus({ submitResponse }) {
11661
+ async _pollReportStatus({ submitResponse }) {
12653
11662
  const pollUrl = "https://reporting.api.bingads.microsoft.com/Reporting/v13/GenerateReport/Poll";
12654
11663
  const submitResponseText = JSON.stringify(submitResponse);
12655
11664
  const pollOpts = {
@@ -12665,7 +11674,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12665
11674
  payload: submitResponseText,
12666
11675
  body: submitResponseText
12667
11676
  };
12668
- return MicrosoftAdsHelper.pollUntilStatus({
11677
+ return await MicrosoftAdsHelper.pollUntilStatus({
12669
11678
  url: pollUrl,
12670
11679
  options: pollOpts,
12671
11680
  isDone: (status) => {
@@ -12678,7 +11687,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12678
11687
  }
12679
11688
  };
12680
11689
  var MicrosoftAdsConnector = class MicrosoftAdsConnector extends AbstractConnector2 {
12681
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
11690
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
12682
11691
  super(config, source, null, runConfig);
12683
11692
  this.storageName = storageName;
12684
11693
  }
@@ -12686,10 +11695,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12686
11695
  * Main method - entry point for the import process
12687
11696
  * Processes all nodes defined in the fields configuration
12688
11697
  */
12689
- startImportProcess() {
11698
+ async startImportProcess() {
12690
11699
  const fields = MicrosoftAdsHelper.parseFields(this.config.Fields.value);
12691
11700
  for (const nodeName in fields) {
12692
- this.processNode({
11701
+ await this.processNode({
12693
11702
  nodeName,
12694
11703
  accountId: this.config.AccountID.value,
12695
11704
  fields: fields[nodeName] || []
@@ -12703,15 +11712,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12703
11712
  * @param {string} options.accountId - Account ID
12704
11713
  * @param {Array<string>} options.fields - Array of fields to fetch
12705
11714
  */
12706
- processNode({ nodeName, accountId, fields }) {
11715
+ async processNode({ nodeName, accountId, fields }) {
12707
11716
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
12708
- this.processTimeSeriesNode({
11717
+ await this.processTimeSeriesNode({
12709
11718
  nodeName,
12710
11719
  accountId,
12711
11720
  fields
12712
11721
  });
12713
11722
  } else {
12714
- this.processCatalogNode({
11723
+ await this.processCatalogNode({
12715
11724
  nodeName,
12716
11725
  accountId,
12717
11726
  fields
@@ -12726,7 +11735,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12726
11735
  * @param {Array<string>} options.fields - Array of fields to fetch
12727
11736
  * @param {Object} options.storage - Storage instance
12728
11737
  */
12729
- processTimeSeriesNode({ nodeName, accountId, fields }) {
11738
+ async processTimeSeriesNode({ nodeName, accountId, fields }) {
12730
11739
  var _a;
12731
11740
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
12732
11741
  if (daysToFetch <= 0) {
@@ -12736,9 +11745,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12736
11745
  for (let dayOffset = 0; dayOffset < daysToFetch; dayOffset++) {
12737
11746
  const currentDate = new Date(startDate);
12738
11747
  currentDate.setDate(currentDate.getDate() + dayOffset);
12739
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
11748
+ const formattedDate = DateUtils2.formatDate(currentDate);
12740
11749
  this.config.logMessage(`Processing ${nodeName} for ${accountId} on ${formattedDate} (day ${dayOffset + 1} of ${daysToFetch})`);
12741
- const data = this.source.fetchData({
11750
+ const data = await this.source.fetchData({
12742
11751
  nodeName,
12743
11752
  accountId,
12744
11753
  start_time: formattedDate,
@@ -12748,7 +11757,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12748
11757
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
12749
11758
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
12750
11759
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
12751
- this.getStorageByNode(nodeName).saveData(preparedData);
11760
+ const storage = await this.getStorageByNode(nodeName);
11761
+ await storage.saveData(preparedData);
12752
11762
  data.length && this.config.logMessage(`Successfully saved ${data.length} rows for ${formattedDate}`);
12753
11763
  }
12754
11764
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
@@ -12764,22 +11774,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12764
11774
  * @param {Array<string>} options.fields - Array of fields to fetch
12765
11775
  * @param {Object} options.storage - Storage instance
12766
11776
  */
12767
- processCatalogNode({ nodeName, accountId, fields }) {
11777
+ async processCatalogNode({ nodeName, accountId, fields }) {
12768
11778
  var _a;
12769
- const data = this.source.fetchData({
11779
+ const data = await this.source.fetchData({
12770
11780
  nodeName,
12771
11781
  accountId,
12772
11782
  fields,
12773
- onBatchReady: (batchData) => {
11783
+ onBatchReady: async (batchData) => {
12774
11784
  this.config.logMessage(`Saving batch of ${batchData.length} records to storage`);
12775
11785
  const preparedData = this.addMissingFieldsToData(batchData, fields);
12776
- this.getStorageByNode(nodeName).saveData(preparedData);
11786
+ const storage = await this.getStorageByNode(nodeName);
11787
+ await storage.saveData(preparedData);
12777
11788
  }
12778
11789
  });
12779
11790
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
12780
11791
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
12781
11792
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
12782
- this.getStorageByNode(nodeName).saveData(preparedData);
11793
+ const storage = await this.getStorageByNode(nodeName);
11794
+ await storage.saveData(preparedData);
12783
11795
  }
12784
11796
  }
12785
11797
  /**
@@ -12787,7 +11799,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12787
11799
  * @param {string} nodeName - Name of the node
12788
11800
  * @returns {Object} Storage instance
12789
11801
  */
12790
- getStorageByNode(nodeName) {
11802
+ async getStorageByNode(nodeName) {
12791
11803
  if (!("storages" in this)) {
12792
11804
  this.storages = {};
12793
11805
  }
@@ -12805,6 +11817,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12805
11817
  this.source.fieldsSchema[nodeName].fields,
12806
11818
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
12807
11819
  );
11820
+ await this.storages[nodeName].init();
12808
11821
  }
12809
11822
  return this.storages[nodeName];
12810
11823
  }
@@ -12821,7 +11834,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
12821
11834
  };
12822
11835
  })();
12823
11836
  const LinkedInPages = (function() {
12824
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
11837
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
12825
11838
  var followerStatisticsTimeBoundFields = {
12826
11839
  "organization_urn": {
12827
11840
  "description": "Organization URN",
@@ -13003,7 +12016,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13003
12016
  * @param {Object} params - Additional parameters for the request
13004
12017
  * @returns {Array} - Array of processed data objects
13005
12018
  */
13006
- fetchData(nodeName, urn, params = {}) {
12019
+ async fetchData(nodeName, urn, params = {}) {
13007
12020
  var _a;
13008
12021
  const fields = params.fields || [];
13009
12022
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -13013,7 +12026,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13013
12026
  }
13014
12027
  switch (nodeName) {
13015
12028
  case "follower_statistics_time_bound":
13016
- return this.fetchOrganizationStats({
12029
+ return await this.fetchOrganizationStats({
13017
12030
  urn,
13018
12031
  nodeName,
13019
12032
  endpoint: "organizationalEntityFollowerStatistics",
@@ -13022,7 +12035,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13022
12035
  params
13023
12036
  });
13024
12037
  case "follower_statistics":
13025
- return this.fetchOrganizationStats({
12038
+ return await this.fetchOrganizationStats({
13026
12039
  urn,
13027
12040
  nodeName,
13028
12041
  endpoint: "organizationalEntityFollowerStatistics",
@@ -13047,7 +12060,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13047
12060
  * @param {Array} [options.params.fields] - Additional parameters including fields
13048
12061
  * @returns {Array} - Processed statistics data
13049
12062
  */
13050
- fetchOrganizationStats(options) {
12063
+ async fetchOrganizationStats(options) {
13051
12064
  const { urn, nodeName, endpoint, entityParam, formatter, params } = options;
13052
12065
  const orgUrn = `urn:li:organization:${urn}`;
13053
12066
  const encodedUrn = encodeURIComponent(orgUrn);
@@ -13058,7 +12071,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13058
12071
  const endTimestamp = new Date(params.endDate).getTime();
13059
12072
  url += `&timeIntervals=(timeRange:(start:${startTimestamp},end:${endTimestamp}),timeGranularityType:DAY)`;
13060
12073
  }
13061
- const response = this.makeRequest(url);
12074
+ const response = await this.makeRequest(url);
13062
12075
  const elements = response.elements || [];
13063
12076
  if (elements.length === 0) {
13064
12077
  return [];
@@ -13075,9 +12088,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13075
12088
  * @param {Object} headers - Optional additional headers
13076
12089
  * @returns {Object} - API response parsed from JSON
13077
12090
  */
13078
- makeRequest(url) {
12091
+ async makeRequest(url) {
13079
12092
  console.log(`LinkedIn Pages API URL:`, url);
13080
- OAuthUtils.getAccessToken({
12093
+ await OAuthUtils.getAccessToken({
13081
12094
  config: this.config,
13082
12095
  tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
13083
12096
  formData: {
@@ -13092,12 +12105,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13092
12105
  "X-RestLi-Protocol-Version": "2.0.0"
13093
12106
  };
13094
12107
  const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
13095
- const response = EnvironmentAdapter2.fetch(authUrl, { headers });
13096
- const result = JSON.parse(response.getContentText());
13097
- if (result.status && result.status >= HTTP_STATUS2.BAD_REQUEST) {
13098
- throw new Error(`LinkedIn API Error: ${result.message || "Unknown error"} (Status: ${result.status})`);
12108
+ const response = await HttpUtils2.fetch(authUrl, { headers });
12109
+ const result = await response.getContentText();
12110
+ const parsedResult = JSON.parse(result);
12111
+ if (parsedResult.status && parsedResult.status >= HTTP_STATUS2.BAD_REQUEST) {
12112
+ throw new Error(`LinkedIn API Error: ${parsedResult.message || "Unknown error"} (Status: ${parsedResult.status})`);
13099
12113
  }
13100
- return result;
12114
+ return parsedResult;
13101
12115
  }
13102
12116
  /**
13103
12117
  * Process time-bound statistics data
@@ -13191,7 +12205,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13191
12205
  }
13192
12206
  };
13193
12207
  var LinkedInPagesConnector = class LinkedInPagesConnector extends AbstractConnector2 {
13194
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
12208
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
13195
12209
  super(config, source, null, runConfig);
13196
12210
  this.storageName = storageName;
13197
12211
  }
@@ -13199,11 +12213,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13199
12213
  * Main method - entry point for the import process
13200
12214
  * Processes all nodes defined in the fields configuration
13201
12215
  */
13202
- startImportProcess() {
12216
+ async startImportProcess() {
13203
12217
  const urns = FormatUtils.parseIds(this.config.OrganizationURNs.value, { prefix: "urn:li:organization:" });
13204
12218
  const dataSources = FormatUtils.parseFields(this.config.Fields.value);
13205
12219
  for (const nodeName in dataSources) {
13206
- this.processNode({
12220
+ await this.processNode({
13207
12221
  nodeName,
13208
12222
  urns,
13209
12223
  fields: dataSources[nodeName] || []
@@ -13217,13 +12231,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13217
12231
  * @param {Array} options.urns - URNs to process
13218
12232
  * @param {Array} options.fields - Fields to fetch
13219
12233
  */
13220
- processNode({ nodeName, urns, fields }) {
12234
+ async processNode({ nodeName, urns, fields }) {
13221
12235
  const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
13222
12236
  const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
13223
12237
  if (isTimeSeriesNode && !dateInfo) {
13224
12238
  return;
13225
12239
  }
13226
- this.fetchAndSaveData({
12240
+ await this.fetchAndSaveData({
13227
12241
  nodeName,
13228
12242
  urns,
13229
12243
  fields,
@@ -13244,7 +12258,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13244
12258
  * @param {string} [options.startDate] - Start date for time series data
13245
12259
  * @param {string} [options.endDate] - End date for time series data
13246
12260
  */
13247
- fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
12261
+ async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
13248
12262
  var _a;
13249
12263
  for (const urn of urns) {
13250
12264
  console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
@@ -13252,11 +12266,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13252
12266
  console.log(`End date is +1 day due to LinkedIn Pages API requirements (to include actual end date in results)`);
13253
12267
  }
13254
12268
  const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
13255
- const data = this.source.fetchData(nodeName, urn, params);
12269
+ const data = await this.source.fetchData(nodeName, urn, params);
13256
12270
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
13257
12271
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
13258
12272
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
13259
- this.getStorageByNode(nodeName).saveData(preparedData);
12273
+ const storage = await this.getStorageByNode(nodeName);
12274
+ await storage.saveData(preparedData);
13260
12275
  }
13261
12276
  }
13262
12277
  }
@@ -13265,7 +12280,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13265
12280
  * @param {string} nodeName - Name of the node
13266
12281
  * @returns {Object} - Storage instance
13267
12282
  */
13268
- getStorageByNode(nodeName) {
12283
+ async getStorageByNode(nodeName) {
13269
12284
  if (!("storages" in this)) {
13270
12285
  this.storages = {};
13271
12286
  }
@@ -13283,6 +12298,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13283
12298
  this.source.fieldsSchema[nodeName]["fields"],
13284
12299
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
13285
12300
  );
12301
+ await this.storages[nodeName].init();
13286
12302
  }
13287
12303
  return this.storages[nodeName];
13288
12304
  }
@@ -13320,7 +12336,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
13320
12336
  };
13321
12337
  })();
13322
12338
  const LinkedInAds = (function() {
13323
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
12339
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
13324
12340
  var creativesFields = {
13325
12341
  "account": {
13326
12342
  "description": "URN identifying the advertising account associated with the creative. This field is read-only.",
@@ -14276,7 +13292,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14276
13292
  * @param {Object} params - Additional parameters for the request
14277
13293
  * @returns {Array} - Array of fetched data objects
14278
13294
  */
14279
- fetchData(nodeName, urn, params = {}) {
13295
+ async fetchData(nodeName, urn, params = {}) {
14280
13296
  var _a;
14281
13297
  const fields = params.fields || [];
14282
13298
  const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
@@ -14286,15 +13302,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14286
13302
  }
14287
13303
  switch (nodeName) {
14288
13304
  case "adAccounts":
14289
- return this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
13305
+ return await this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
14290
13306
  case "adCampaignGroups":
14291
- return this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
13307
+ return await this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
14292
13308
  case "adCampaigns":
14293
- return this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
13309
+ return await this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
14294
13310
  case "creatives":
14295
- return this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
13311
+ return await this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
14296
13312
  case "adAnalytics":
14297
- return this.fetchAdAnalytics(urn, params);
13313
+ return await this.fetchAdAnalytics(urn, params);
14298
13314
  default:
14299
13315
  throw new Error(`Unknown node: ${nodeName}`);
14300
13316
  }
@@ -14307,10 +13323,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14307
13323
  * @param {Object} options.params - Additional parameters for the request
14308
13324
  * @returns {Array} - Array containing the single resource
14309
13325
  */
14310
- fetchSingleResource({ urn, resourceType, params }) {
13326
+ async fetchSingleResource({ urn, resourceType, params }) {
14311
13327
  let url = `${this.BASE_URL}${resourceType}/${encodeURIComponent(urn)}`;
14312
13328
  url += `?fields=${this.formatFields(params.fields)}`;
14313
- const result = this.makeRequest(url);
13329
+ const result = await this.makeRequest(url);
14314
13330
  return [result];
14315
13331
  }
14316
13332
  /**
@@ -14322,10 +13338,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14322
13338
  * @param {string} [options.queryType='search'] - Query type parameter
14323
13339
  * @returns {Array} - Array of fetched resources
14324
13340
  */
14325
- fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
13341
+ async fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
14326
13342
  let url = `${this.BASE_URL}adAccounts/${encodeURIComponent(urn)}/${resourceType}?q=${queryType}&pageSize=100`;
14327
13343
  url += `&fields=${this.formatFields(params.fields)}`;
14328
- return this.fetchWithPagination(url);
13344
+ return await this.fetchWithPagination(url);
14329
13345
  }
14330
13346
  /**
14331
13347
  * Fetch analytics data, handling field limits and data merging
@@ -14336,7 +13352,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14336
13352
  * @param {Array} params.fields - Fields to fetch
14337
13353
  * @returns {Array} - Combined array of analytics data
14338
13354
  */
14339
- fetchAdAnalytics(urn, params) {
13355
+ async fetchAdAnalytics(urn, params) {
14340
13356
  const startDate = new Date(params.startDate);
14341
13357
  const endDate = new Date(params.endDate);
14342
13358
  const accountUrn = `urn:li:sponsoredAccount:${urn}`;
@@ -14351,7 +13367,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14351
13367
  encodedUrn,
14352
13368
  fields: fieldChunk
14353
13369
  });
14354
- const res = this.makeRequest(url);
13370
+ const res = await this.makeRequest(url);
14355
13371
  const elements = res.elements || [];
14356
13372
  allResults = this.mergeAnalyticsResults(allResults, elements);
14357
13373
  }
@@ -14480,9 +13496,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14480
13496
  * @param {Object} headers - Optional additional headers
14481
13497
  * @returns {Object} - API response parsed from JSON
14482
13498
  */
14483
- makeRequest(url) {
13499
+ async makeRequest(url) {
14484
13500
  console.log(`LinkedIn Ads API Request URL:`, url);
14485
- OAuthUtils.getAccessToken({
13501
+ await OAuthUtils.getAccessToken({
14486
13502
  config: this.config,
14487
13503
  tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
14488
13504
  formData: {
@@ -14497,8 +13513,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14497
13513
  "X-RestLi-Protocol-Version": "2.0.0"
14498
13514
  };
14499
13515
  const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
14500
- const response = EnvironmentAdapter2.fetch(authUrl, { headers });
14501
- const result = JSON.parse(response.getContentText());
13516
+ const response = await HttpUtils2.fetch(authUrl, { headers });
13517
+ const text = await response.getContentText();
13518
+ const result = JSON.parse(text);
14502
13519
  return result;
14503
13520
  }
14504
13521
  /**
@@ -14507,7 +13524,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14507
13524
  * @param {Object} headers - Optional additional headers
14508
13525
  * @returns {Array} - Combined array of results from all pages
14509
13526
  */
14510
- fetchWithPagination(baseUrl) {
13527
+ async fetchWithPagination(baseUrl) {
14511
13528
  let allResults = [];
14512
13529
  let pageToken = null;
14513
13530
  do {
@@ -14515,7 +13532,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14515
13532
  if (pageToken) {
14516
13533
  pageUrl += `${pageUrl.includes("?") ? "&" : "?"}pageToken=${encodeURIComponent(pageToken)}`;
14517
13534
  }
14518
- const res = this.makeRequest(pageUrl);
13535
+ const res = await this.makeRequest(pageUrl);
14519
13536
  const elements = res.elements || [];
14520
13537
  allResults = allResults.concat(elements);
14521
13538
  const metadata = res.metadata || {};
@@ -14525,7 +13542,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14525
13542
  }
14526
13543
  };
14527
13544
  var LinkedInAdsConnector = class LinkedInAdsConnector extends AbstractConnector2 {
14528
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
13545
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
14529
13546
  super(config, source, null, runConfig);
14530
13547
  this.storageName = storageName;
14531
13548
  }
@@ -14533,11 +13550,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14533
13550
  * Main method - entry point for the import process
14534
13551
  * Processes all nodes defined in the fields configuration
14535
13552
  */
14536
- startImportProcess() {
13553
+ async startImportProcess() {
14537
13554
  const urns = FormatUtils.parseIds(this.config.AccountURNs.value, { prefix: "urn:li:sponsoredAccount:" });
14538
13555
  const dataSources = FormatUtils.parseFields(this.config.Fields.value);
14539
13556
  for (const nodeName in dataSources) {
14540
- this.processNode({
13557
+ await this.processNode({
14541
13558
  nodeName,
14542
13559
  urns,
14543
13560
  fields: dataSources[nodeName] || []
@@ -14551,13 +13568,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14551
13568
  * @param {Array} options.urns - URNs to process
14552
13569
  * @param {Array} options.fields - Fields to fetch
14553
13570
  */
14554
- processNode({ nodeName, urns, fields }) {
13571
+ async processNode({ nodeName, urns, fields }) {
14555
13572
  const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
14556
13573
  const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
14557
13574
  if (isTimeSeriesNode && !dateInfo) {
14558
13575
  return;
14559
13576
  }
14560
- this.fetchAndSaveData({
13577
+ await this.fetchAndSaveData({
14561
13578
  nodeName,
14562
13579
  urns,
14563
13580
  fields,
@@ -14578,16 +13595,17 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14578
13595
  * @param {string} [options.startDate] - Start date for time series data
14579
13596
  * @param {string} [options.endDate] - End date for time series data
14580
13597
  */
14581
- fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
13598
+ async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
14582
13599
  var _a;
14583
13600
  for (const urn of urns) {
14584
13601
  console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
14585
13602
  const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
14586
- const data = this.source.fetchData(nodeName, urn, params);
13603
+ const data = await this.source.fetchData(nodeName, urn, params);
14587
13604
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
14588
13605
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
14589
13606
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
14590
- this.getStorageByNode(nodeName).saveData(preparedData);
13607
+ const storage = await this.getStorageByNode(nodeName);
13608
+ await storage.saveData(preparedData);
14591
13609
  }
14592
13610
  }
14593
13611
  }
@@ -14596,7 +13614,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14596
13614
  * @param {string} nodeName - Name of the node
14597
13615
  * @returns {Object} - Storage instance
14598
13616
  */
14599
- getStorageByNode(nodeName) {
13617
+ async getStorageByNode(nodeName) {
14600
13618
  if (!("storages" in this)) {
14601
13619
  this.storages = {};
14602
13620
  }
@@ -14614,6 +13632,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14614
13632
  this.source.fieldsSchema[nodeName]["fields"],
14615
13633
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
14616
13634
  );
13635
+ await this.storages[nodeName].init();
14617
13636
  }
14618
13637
  return this.storages[nodeName];
14619
13638
  }
@@ -14650,7 +13669,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
14650
13669
  };
14651
13670
  })();
14652
13671
  const GoogleAds = (function() {
14653
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
13672
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
14654
13673
  var keywordStatsFields = {
14655
13674
  "keyword_id": {
14656
13675
  "description": "Keyword Criterion ID",
@@ -15551,7 +14570,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15551
14570
  * Get access token based on authentication type
15552
14571
  * Supports OAuth2 and Service Account authentication
15553
14572
  */
15554
- getAccessToken() {
14573
+ async getAccessToken() {
15555
14574
  var _a;
15556
14575
  if (this.accessToken && this.tokenExpiryTime && Date.now() < this.tokenExpiryTime) {
15557
14576
  return this.accessToken;
@@ -15564,7 +14583,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15564
14583
  let accessToken;
15565
14584
  try {
15566
14585
  if (authType === "oauth2") {
15567
- accessToken = OAuthUtils.getAccessToken({
14586
+ accessToken = await OAuthUtils.getAccessToken({
15568
14587
  config: this.config,
15569
14588
  tokenUrl: "https://oauth2.googleapis.com/token",
15570
14589
  formData: {
@@ -15575,7 +14594,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15575
14594
  }
15576
14595
  });
15577
14596
  } else if (authType === "service_account") {
15578
- accessToken = OAuthUtils.getServiceAccountToken({
14597
+ accessToken = await OAuthUtils.getServiceAccountToken({
15579
14598
  config: this.config,
15580
14599
  tokenUrl: "https://oauth2.googleapis.com/token",
15581
14600
  serviceAccountKeyJson: authConfig.ServiceAccountKey.value,
@@ -15602,11 +14621,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15602
14621
  * @param {Date} [options.startDate] - Start date for time series data
15603
14622
  * @returns {Array<Object>} - Fetched data
15604
14623
  */
15605
- fetchData(nodeName, customerId, options) {
14624
+ async fetchData(nodeName, customerId, options) {
15606
14625
  console.log("Fetching data from Google Ads API for customer:", customerId);
15607
14626
  const { fields, startDate } = options;
15608
14627
  const query = this._buildQuery({ nodeName, fields, startDate });
15609
- return this.makeRequest({ customerId, query, nodeName, fields });
14628
+ const response = await this.makeRequest({ customerId, query, nodeName, fields });
14629
+ return await response;
15610
14630
  }
15611
14631
  /**
15612
14632
  * Convert field names to API field names
@@ -15656,7 +14676,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15656
14676
  const resourceName = this._getResourceName(nodeName);
15657
14677
  let query = `SELECT ${apiFields.join(", ")} FROM ${resourceName}`;
15658
14678
  if (startDate && this.fieldsSchema[nodeName].isTimeSeries) {
15659
- const formattedDate = EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd");
14679
+ const formattedDate = DateUtils2.formatDate(startDate);
15660
14680
  query += ` WHERE segments.date = '${formattedDate}'`;
15661
14681
  }
15662
14682
  return query;
@@ -15670,9 +14690,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15670
14690
  * @param {Array<string>} options.fields - Fields that were requested
15671
14691
  * @returns {Array<Object>} - API response data
15672
14692
  */
15673
- makeRequest({ customerId, query, nodeName, fields }) {
14693
+ async makeRequest({ customerId, query, nodeName, fields }) {
15674
14694
  var _a, _b;
15675
- const accessToken = this.getAccessToken();
14695
+ const accessToken = await this.getAccessToken();
15676
14696
  const url = `https://googleads.googleapis.com/v21/customers/${customerId}/googleAds:search`;
15677
14697
  console.log(`Google Ads API Request URL: ${url}`);
15678
14698
  console.log(`GAQL Query: ${query}`);
@@ -15696,8 +14716,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15696
14716
  body: JSON.stringify(requestBody),
15697
14717
  muteHttpExceptions: true
15698
14718
  };
15699
- const response = this.urlFetchWithRetry(url, options);
15700
- const jsonData = JSON.parse(response.getContentText());
14719
+ const response = await this.urlFetchWithRetry(url, options);
14720
+ const text = await response.getContentText();
14721
+ const jsonData = JSON.parse(text);
15701
14722
  if (jsonData.error) {
15702
14723
  throw new Error(`Google Ads API error: ${jsonData.error.message}`);
15703
14724
  }
@@ -15752,7 +14773,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15752
14773
  }
15753
14774
  };
15754
14775
  var GoogleAdsConnector = class GoogleAdsConnector extends AbstractConnector2 {
15755
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
14776
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
15756
14777
  super(config, source, null, runConfig);
15757
14778
  this.storageName = storageName;
15758
14779
  }
@@ -15760,11 +14781,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15760
14781
  * Main method - entry point for the import process
15761
14782
  * Processes all nodes defined in the fields configuration
15762
14783
  */
15763
- startImportProcess() {
14784
+ async startImportProcess() {
15764
14785
  const customerIds = FormatUtils.parseIds(this.config.CustomerId.value, { stripCharacters: "-" });
15765
14786
  const fields = FormatUtils.parseFields(this.config.Fields.value);
15766
14787
  for (const nodeName in fields) {
15767
- this.processNode({
14788
+ await this.processNode({
15768
14789
  nodeName,
15769
14790
  customerIds,
15770
14791
  fields: fields[nodeName] || []
@@ -15778,16 +14799,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15778
14799
  * @param {Array<string>} options.customerIds - Array of customer IDs to process
15779
14800
  * @param {Array<string>} options.fields - Array of fields to fetch
15780
14801
  */
15781
- processNode({ nodeName, customerIds, fields }) {
14802
+ async processNode({ nodeName, customerIds, fields }) {
15782
14803
  for (const customerId of customerIds) {
15783
14804
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
15784
- this.processTimeSeriesNode({
14805
+ await this.processTimeSeriesNode({
15785
14806
  nodeName,
15786
14807
  customerId,
15787
14808
  fields
15788
14809
  });
15789
14810
  } else {
15790
- this.processCatalogNode({
14811
+ await this.processCatalogNode({
15791
14812
  nodeName,
15792
14813
  customerId,
15793
14814
  fields
@@ -15802,7 +14823,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15802
14823
  * @param {string} options.customerId - Customer ID
15803
14824
  * @param {Array<string>} options.fields - Array of fields to fetch
15804
14825
  */
15805
- processTimeSeriesNode({ nodeName, customerId, fields }) {
14826
+ async processTimeSeriesNode({ nodeName, customerId, fields }) {
15806
14827
  var _a;
15807
14828
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
15808
14829
  if (daysToFetch <= 0) {
@@ -15812,12 +14833,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15812
14833
  for (let i = 0; i < daysToFetch; i++) {
15813
14834
  const currentDate = new Date(startDate);
15814
14835
  currentDate.setDate(currentDate.getDate() + i);
15815
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
15816
- const data = this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
14836
+ const formattedDate = DateUtils2.formatDate(currentDate);
14837
+ const data = await this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
15817
14838
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId} on ${formattedDate}` : `ℹ️ No records have been fetched`);
15818
14839
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
15819
14840
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
15820
- this.getStorageByNode(nodeName).saveData(preparedData);
14841
+ const storage = await this.getStorageByNode(nodeName);
14842
+ await storage.saveData(preparedData);
15821
14843
  }
15822
14844
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
15823
14845
  this.config.updateLastRequstedDate(currentDate);
@@ -15831,13 +14853,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15831
14853
  * @param {string} options.customerId - Customer ID
15832
14854
  * @param {Array<string>} options.fields - Array of fields to fetch
15833
14855
  */
15834
- processCatalogNode({ nodeName, customerId, fields }) {
14856
+ async processCatalogNode({ nodeName, customerId, fields }) {
15835
14857
  var _a;
15836
- const data = this.source.fetchData(nodeName, customerId, { fields });
14858
+ const data = await this.source.fetchData(nodeName, customerId, { fields });
15837
14859
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId}` : `ℹ️ No records have been fetched`);
15838
14860
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
15839
14861
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
15840
- this.getStorageByNode(nodeName).saveData(preparedData);
14862
+ const storage = await this.getStorageByNode(nodeName);
14863
+ await storage.saveData(preparedData);
15841
14864
  }
15842
14865
  }
15843
14866
  /**
@@ -15845,7 +14868,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15845
14868
  * @param {string} nodeName - Name of the node
15846
14869
  * @returns {Object} - Storage instance
15847
14870
  */
15848
- getStorageByNode(nodeName) {
14871
+ async getStorageByNode(nodeName) {
15849
14872
  if (!("storages" in this)) {
15850
14873
  this.storages = {};
15851
14874
  }
@@ -15863,6 +14886,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15863
14886
  this.source.fieldsSchema[nodeName].fields,
15864
14887
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
15865
14888
  );
14889
+ await this.storages[nodeName].init();
15866
14890
  }
15867
14891
  return this.storages[nodeName];
15868
14892
  }
@@ -15879,7 +14903,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
15879
14903
  };
15880
14904
  })();
15881
14905
  const GitHub = (function() {
15882
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
14906
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
15883
14907
  var repositoryStatsFields = {
15884
14908
  date: {
15885
14909
  type: "date",
@@ -16192,16 +15216,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16192
15216
  * @param {Array<string>} opts.fields
16193
15217
  * @returns {Array<Object>}
16194
15218
  */
16195
- fetchData({ nodeName, fields = [] }) {
15219
+ async fetchData({ nodeName, fields = [] }) {
16196
15220
  switch (nodeName) {
16197
15221
  case "repository":
16198
- const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
15222
+ const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
16199
15223
  return this._filterBySchema({ items: [repoData], nodeName, fields });
16200
15224
  case "contributors":
16201
- const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
15225
+ const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
16202
15226
  return this._filterBySchema({ items: contribData, nodeName, fields });
16203
15227
  case "repositoryStats":
16204
- return this._fetchRepositoryStats({ nodeName, fields });
15228
+ return await this._fetchRepositoryStats({ nodeName, fields });
16205
15229
  default:
16206
15230
  throw new Error(`Unknown node: ${nodeName}`);
16207
15231
  }
@@ -16213,9 +15237,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16213
15237
  * @param {Array<string>} options.fields - Array of fields to fetch
16214
15238
  * @returns {Array} Array of repository statistics data
16215
15239
  */
16216
- _fetchRepositoryStats({ nodeName, fields }) {
16217
- const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
16218
- const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
15240
+ async _fetchRepositoryStats({ nodeName, fields }) {
15241
+ const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
15242
+ const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
16219
15243
  return this._filterBySchema({
16220
15244
  items: [{
16221
15245
  "date": new Date((/* @__PURE__ */ new Date()).setHours(0, 0, 0, 0)),
@@ -16232,30 +15256,32 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16232
15256
  * @param {string} options.endpoint - API endpoint path (e.g., "repos/owner/repo")
16233
15257
  * @returns {Object} - API response parsed from JSON
16234
15258
  */
16235
- makeRequest({ endpoint }) {
16236
- const baseUrl = "https://api.github.com/";
16237
- const url = `${baseUrl}${endpoint}`;
16238
- const response = EnvironmentAdapter2.fetch(url, {
16239
- "method": "get",
16240
- "muteHttpExceptions": true,
16241
- "headers": {
16242
- "Accept": "application/vnd.github+json",
16243
- "Authorization": `Bearer ${this.config.AccessToken.value}`,
16244
- "User-Agent": "owox"
15259
+ async makeRequest({ endpoint }) {
15260
+ try {
15261
+ const baseUrl = "https://api.github.com/";
15262
+ const url = `${baseUrl}${endpoint}`;
15263
+ const response = await HttpUtils2.fetch(url, {
15264
+ "method": "get",
15265
+ "muteHttpExceptions": true,
15266
+ "headers": {
15267
+ "Accept": "application/vnd.github+json",
15268
+ "Authorization": `Bearer ${this.config.AccessToken.value}`,
15269
+ "User-Agent": "owox"
15270
+ }
15271
+ });
15272
+ const text = await response.getContentText();
15273
+ const result = JSON.parse(text);
15274
+ if (result && result.message === "Not Found") {
15275
+ throw new Error(
15276
+ "The repository was not found. The repository name should be in the format: owner/repo"
15277
+ );
16245
15278
  }
16246
- });
16247
- const result = JSON.parse(response.getContentText());
16248
- if (result && result.message === "Not Found") {
16249
- throw new Error(
16250
- "The repository was not found. The repository name should be in the format: owner/repo"
16251
- );
15279
+ return result;
15280
+ } catch (error) {
15281
+ this.config.logMessage(`Error: ${error.message}`);
15282
+ console.error(error.stack);
15283
+ throw error;
16252
15284
  }
16253
- return result;
16254
- }
16255
- catch(error) {
16256
- this.config.logMessage(`Error: ${error.message}`);
16257
- console.error(error.stack);
16258
- throw error;
16259
15285
  }
16260
15286
  /**
16261
15287
  * Keep only requestedFields plus any schema-required keys.
@@ -16281,7 +15307,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16281
15307
  }
16282
15308
  };
16283
15309
  var GitHubConnector = class GitHubConnector extends AbstractConnector2 {
16284
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
15310
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
16285
15311
  super(config, source, null, runConfig);
16286
15312
  this.storageName = storageName;
16287
15313
  }
@@ -16289,10 +15315,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16289
15315
  * Main method - entry point for the import process
16290
15316
  * Processes all nodes defined in the fields configuration
16291
15317
  */
16292
- startImportProcess() {
15318
+ async startImportProcess() {
16293
15319
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
16294
15320
  for (const nodeName in fields) {
16295
- this.processNode({
15321
+ await this.processNode({
16296
15322
  nodeName,
16297
15323
  fields: fields[nodeName] || []
16298
15324
  });
@@ -16304,11 +15330,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16304
15330
  * @param {string} options.nodeName - Name of the node to process
16305
15331
  * @param {Array<string>} options.fields - Array of fields to fetch
16306
15332
  */
16307
- processNode({ nodeName, fields }) {
15333
+ async processNode({ nodeName, fields }) {
16308
15334
  if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
16309
- this.processTimeSeriesNode({ nodeName, fields });
15335
+ await this.processTimeSeriesNode({ nodeName, fields });
16310
15336
  } else {
16311
- this.processCatalogNode({ nodeName, fields });
15337
+ await this.processCatalogNode({ nodeName, fields });
16312
15338
  }
16313
15339
  }
16314
15340
  /**
@@ -16318,7 +15344,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16318
15344
  * @param {Array<string>} options.fields - Array of fields to fetch
16319
15345
  * @param {Object} options.storage - Storage instance
16320
15346
  */
16321
- processTimeSeriesNode({ nodeName, fields }) {
15347
+ async processTimeSeriesNode({ nodeName, fields }) {
16322
15348
  console.log(`Time series node processing not implemented for ${nodeName}`);
16323
15349
  }
16324
15350
  /**
@@ -16328,13 +15354,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16328
15354
  * @param {Array<string>} options.fields - Array of fields to fetch
16329
15355
  * @param {Object} options.storage - Storage instance
16330
15356
  */
16331
- processCatalogNode({ nodeName, fields }) {
15357
+ async processCatalogNode({ nodeName, fields }) {
16332
15358
  var _a;
16333
- const data = this.source.fetchData({ nodeName, fields });
15359
+ const data = await this.source.fetchData({ nodeName, fields });
16334
15360
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched` : `No records have been fetched`);
16335
15361
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
16336
15362
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
16337
- this.getStorageByNode(nodeName).saveData(preparedData);
15363
+ const storage = await this.getStorageByNode(nodeName);
15364
+ await storage.saveData(preparedData);
16338
15365
  }
16339
15366
  }
16340
15367
  /**
@@ -16342,7 +15369,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16342
15369
  * @param {string} nodeName - Name of the node
16343
15370
  * @returns {Object} Storage instance
16344
15371
  */
16345
- getStorageByNode(nodeName) {
15372
+ async getStorageByNode(nodeName) {
16346
15373
  if (!("storages" in this)) {
16347
15374
  this.storages = {};
16348
15375
  }
@@ -16360,6 +15387,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16360
15387
  this.source.fieldsSchema[nodeName].fields,
16361
15388
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
16362
15389
  );
15390
+ await this.storages[nodeName].init();
16363
15391
  }
16364
15392
  return this.storages[nodeName];
16365
15393
  }
@@ -16376,7 +15404,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
16376
15404
  };
16377
15405
  })();
16378
15406
  const FacebookMarketing = (function() {
16379
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
15407
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
16380
15408
  var adGroupFields = {
16381
15409
  "id": {
16382
15410
  "description": "The ID of this ad.",
@@ -20881,12 +19909,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
20881
19909
  @return data array
20882
19910
 
20883
19911
  */
20884
- fetchData(nodeName, accountId, fields, startDate = null) {
19912
+ async fetchData(nodeName, accountId, fields, startDate = null) {
20885
19913
  let url = "https://graph.facebook.com/v23.0/";
20886
19914
  let formattedDate = null;
20887
19915
  let timeRange = null;
20888
19916
  if (startDate) {
20889
- formattedDate = EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd");
19917
+ formattedDate = DateUtils2.formatDate(startDate);
20890
19918
  timeRange = encodeURIComponent(JSON.stringify({ since: formattedDate, until: formattedDate }));
20891
19919
  }
20892
19920
  switch (nodeName) {
@@ -20910,7 +19938,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
20910
19938
  case "ad-account/insights-by-region":
20911
19939
  case "ad-account/insights-by-product-id":
20912
19940
  case "ad-account/insights-by-age-and-gender":
20913
- return this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
19941
+ return await this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
20914
19942
  case "ad-group":
20915
19943
  url += `act_${accountId}/ads?fields=${this._buildFieldsString({ nodeName, fields })}&limit=${this.fieldsSchema[nodeName].limit}`;
20916
19944
  break;
@@ -20919,7 +19947,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
20919
19947
  }
20920
19948
  console.log(`Facebook API URL:`, url);
20921
19949
  url += `&access_token=${this.config.AccessToken.value}`;
20922
- return this._fetchPaginatedData(url, nodeName, fields);
19950
+ return await this._fetchPaginatedData(url, nodeName, fields);
20923
19951
  }
20924
19952
  //---- castRecordFields -------------------------------------------------
20925
19953
  /**
@@ -20981,7 +20009,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
20981
20009
  * @return {Array} Processed insights data
20982
20010
  * @private
20983
20011
  */
20984
- _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
20012
+ async _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
20985
20013
  const breakdowns = this.fieldsSchema[nodeName].breakdowns || [];
20986
20014
  const regularFields = this._prepareFields({ nodeName, fields, breakdowns });
20987
20015
  const requestUrl = this._buildInsightsUrl({
@@ -20992,7 +20020,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
20992
20020
  nodeName,
20993
20021
  url
20994
20022
  });
20995
- const allData = this._fetchPaginatedData(requestUrl, nodeName, fields);
20023
+ const allData = await this._fetchPaginatedData(requestUrl, nodeName, fields);
20996
20024
  if (this.config.ProcessShortLinks.value && allData.length > 0 && allData.some((record) => record.link_url_asset)) {
20997
20025
  return processShortLinks(allData, {
20998
20026
  shortLinkField: "link_url_asset",
@@ -21133,12 +20161,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21133
20161
  * @return {Array} All fetched data
21134
20162
  * @private
21135
20163
  */
21136
- _fetchPaginatedData(initialUrl, nodeName, fields) {
20164
+ async _fetchPaginatedData(initialUrl, nodeName, fields) {
21137
20165
  var allData = [];
21138
20166
  var nextPageURL = initialUrl;
21139
20167
  while (nextPageURL) {
21140
- var response = this.urlFetchWithRetry(nextPageURL);
21141
- var jsonData = JSON.parse(response.getContentText());
20168
+ var response = await this.urlFetchWithRetry(nextPageURL);
20169
+ var text = await response.getContentText();
20170
+ var jsonData = JSON.parse(text);
21142
20171
  if ("data" in jsonData) {
21143
20172
  nextPageURL = jsonData.paging ? jsonData.paging.next : null;
21144
20173
  jsonData.data.forEach((record, index) => {
@@ -21161,12 +20190,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21161
20190
  };
21162
20191
  var FacebookMarketingConnector = class FacebookMarketingConnector extends AbstractConnector2 {
21163
20192
  // ---- constructor ------------------------------------
21164
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
20193
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
21165
20194
  super(config, source, null, runConfig);
21166
20195
  this.storageName = storageName;
21167
20196
  }
21168
20197
  //---- startImportProcess -------------------------------------------------
21169
- startImportProcess() {
20198
+ async startImportProcess() {
21170
20199
  let accountsIds = String(this.config.AccoundIDs.value).split(/[,;]\s*/);
21171
20200
  let fields = this.config.Fields.value.split(", ").reduce((acc, pair) => {
21172
20201
  let [key, value] = pair.split(" ");
@@ -21178,7 +20207,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21178
20207
  if (nodeName in this.source.fieldsSchema && this.source.fieldsSchema[nodeName].isTimeSeries) {
21179
20208
  timeSeriesNodes[nodeName] = fields[nodeName];
21180
20209
  } else {
21181
- this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
20210
+ await this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
21182
20211
  }
21183
20212
  }
21184
20213
  if (Object.keys(timeSeriesNodes).length > 0) {
@@ -21186,27 +20215,28 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21186
20215
  let daysToFetch = null;
21187
20216
  [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
21188
20217
  if (daysToFetch > 0) {
21189
- this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
20218
+ await this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
21190
20219
  }
21191
20220
  }
21192
20221
  }
21193
20222
  //---- startImportProcessOfCatalogData -------------------------------------------------
21194
20223
  /*
21195
20224
 
21196
- Imports catalog (not time seriesed) data
20225
+ Imports catalog (not time seriesed) data
21197
20226
 
21198
20227
  @param nodeName string Node name
21199
20228
  @param accountsIds array list of account ids
21200
- @param fields array list of fields
20229
+ @param fields array list of fields
21201
20230
 
21202
20231
  */
21203
- startImportProcessOfCatalogData(nodeName, accountIds, fields) {
20232
+ async startImportProcessOfCatalogData(nodeName, accountIds, fields) {
21204
20233
  var _a;
21205
20234
  for (var i in accountIds) {
21206
20235
  let accountId = accountIds[i];
21207
- let data = this.source.fetchData(nodeName, accountId, fields);
20236
+ let data = await this.source.fetchData(nodeName, accountId, fields);
21208
20237
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
21209
- this.getStorageByNode(nodeName).saveData(data);
20238
+ const storage = await this.getStorageByNode(nodeName);
20239
+ await storage.saveData(data);
21210
20240
  }
21211
20241
  data.length && this.config.logMessage(`${data.length} rows of ${nodeName} were fetched for account ${accountId}`);
21212
20242
  }
@@ -21214,23 +20244,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21214
20244
  //---- startImportProcessOfTimeSeriesData -------------------------------------------------
21215
20245
  /*
21216
20246
 
21217
- Imports time series (not catalog) data
20247
+ Imports time series (not catalog) data
21218
20248
 
21219
20249
  @param accountsIds (array) list of account ids
21220
20250
  @param timeSeriesNodes (object) of properties, each is array of fields
21221
- @param startDate (Data) start date
20251
+ @param startDate (Data) start date
21222
20252
  @param daysToFetch (integer) days to import
21223
20253
 
21224
20254
  */
21225
- startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
20255
+ async startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
21226
20256
  var _a;
21227
20257
  for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
21228
20258
  for (let accountId of accountsIds) {
21229
20259
  for (var nodeName in timeSeriesNodes) {
21230
- this.config.logMessage(`Start importing data for ${EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd")}: ${accountId}/${nodeName}`);
21231
- let data = this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
20260
+ this.config.logMessage(`Start importing data for ${DateUtils2.formatDate(startDate)}: ${accountId}/${nodeName}`);
20261
+ let data = await this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
21232
20262
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
21233
- this.getStorageByNode(nodeName).saveData(data);
20263
+ const storage = await this.getStorageByNode(nodeName);
20264
+ await storage.saveData(data);
21234
20265
  }
21235
20266
  this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
21236
20267
  }
@@ -21250,7 +20281,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21250
20281
  * @return AbstractStorage
21251
20282
  *
21252
20283
  */
21253
- getStorageByNode(nodeName) {
20284
+ async getStorageByNode(nodeName) {
21254
20285
  if (!("storages" in this)) {
21255
20286
  this.storages = {};
21256
20287
  }
@@ -21268,6 +20299,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21268
20299
  this.source.fieldsSchema[nodeName]["fields"],
21269
20300
  `${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
21270
20301
  );
20302
+ await this.storages[nodeName].init();
21271
20303
  }
21272
20304
  return this.storages[nodeName];
21273
20305
  }
@@ -21284,7 +20316,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
21284
20316
  };
21285
20317
  })();
21286
20318
  const CriteoAds = (function() {
21287
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
20319
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
21288
20320
  var CriteoAdsHelper = {
21289
20321
  /**
21290
20322
  * Parse fields string into a structured object
@@ -22131,10 +21163,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22131
21163
  * @param {Date} opts.date
22132
21164
  * @returns {Array<Object>}
22133
21165
  */
22134
- fetchData({ nodeName, accountId, fields = [], date }) {
21166
+ async fetchData({ nodeName, accountId, fields = [], date }) {
22135
21167
  switch (nodeName) {
22136
21168
  case "statistics":
22137
- return this._fetchStatistics({ accountId, fields, date });
21169
+ return await this._fetchStatistics({ accountId, fields, date });
22138
21170
  default:
22139
21171
  throw new Error(`Unknown node: ${nodeName}`);
22140
21172
  }
@@ -22148,16 +21180,17 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22148
21180
  * @returns {Array<Object>} - Parsed and enriched data
22149
21181
  * @private
22150
21182
  */
22151
- _fetchStatistics({ accountId, fields, date }) {
21183
+ async _fetchStatistics({ accountId, fields, date }) {
22152
21184
  const uniqueKeys = this.fieldsSchema.statistics.uniqueKeys || [];
22153
21185
  const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
22154
21186
  if (missingKeys.length > 0) {
22155
21187
  throw new Error(`Missing required unique fields for endpoint 'statistics'. Missing fields: ${missingKeys.join(", ")}`);
22156
21188
  }
22157
- this.getAccessToken();
21189
+ await this.getAccessToken();
22158
21190
  const requestBody = this._buildStatisticsRequestBody({ accountId, fields, date });
22159
- const response = this._makeApiRequest(requestBody);
22160
- const jsonObject = JSON.parse(response.getContentText());
21191
+ const response = await this._makeApiRequest(requestBody);
21192
+ const text = await response.getContentText();
21193
+ const jsonObject = JSON.parse(text);
22161
21194
  return this.parseApiResponse({ apiResponse: jsonObject, date, accountId, fields });
22162
21195
  }
22163
21196
  /**
@@ -22195,7 +21228,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22195
21228
  * @returns {Object} - HTTP response
22196
21229
  * @private
22197
21230
  */
22198
- _makeApiRequest(requestBody) {
21231
+ async _makeApiRequest(requestBody) {
22199
21232
  const apiVersion = "2025-07";
22200
21233
  const apiUrl = `https://api.criteo.com/${apiVersion}/statistics/report`;
22201
21234
  const options = {
@@ -22209,19 +21242,20 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22209
21242
  body: JSON.stringify(requestBody)
22210
21243
  // TODO: body is for Node.js; refactor to centralize JSON option creation
22211
21244
  };
22212
- const response = this.urlFetchWithRetry(apiUrl, options);
21245
+ const response = await this.urlFetchWithRetry(apiUrl, options);
22213
21246
  const responseCode = response.getResponseCode();
22214
21247
  if (responseCode === HTTP_STATUS2.OK) {
22215
21248
  return response;
22216
21249
  } else {
22217
- throw new Error(`API Error (${responseCode}): ${response.getContentText()}`);
21250
+ const text = await response.getContentText();
21251
+ throw new Error(`API Error (${responseCode}): ${text}`);
22218
21252
  }
22219
21253
  }
22220
21254
  /**
22221
21255
  * Get access token from API
22222
21256
  * Docs: https://developers.criteo.com/marketing-solutions/docs/authorization-code-setup
22223
21257
  */
22224
- getAccessToken() {
21258
+ async getAccessToken() {
22225
21259
  var _a;
22226
21260
  if ((_a = this.config.AccessToken) == null ? void 0 : _a.value) {
22227
21261
  return;
@@ -22244,8 +21278,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22244
21278
  muteHttpExceptions: true
22245
21279
  };
22246
21280
  try {
22247
- const response = this.urlFetchWithRetry(tokenUrl, options);
22248
- const responseData = JSON.parse(response.getContentText());
21281
+ const response = await this.urlFetchWithRetry(tokenUrl, options);
21282
+ const text = await response.getContentText();
21283
+ const responseData = JSON.parse(text);
22249
21284
  const accessToken = responseData["access_token"];
22250
21285
  this.config.AccessToken = {
22251
21286
  value: accessToken
@@ -22293,20 +21328,20 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22293
21328
  }
22294
21329
  };
22295
21330
  var CriteoAdsConnector = class CriteoAdsConnector extends AbstractConnector2 {
22296
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
21331
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
22297
21332
  super(config, source, null, runConfig);
22298
21333
  this.storageName = storageName;
22299
21334
  }
22300
21335
  /**
22301
21336
  * Main method - entry point for the import process
22302
21337
  */
22303
- startImportProcess() {
21338
+ async startImportProcess() {
22304
21339
  var _a, _b;
22305
21340
  const fields = CriteoAdsHelper.parseFields(((_a = this.config.Fields) == null ? void 0 : _a.value) || "");
22306
21341
  const advertiserIds = CriteoAdsHelper.parseAdvertiserIds(((_b = this.config.AdvertiserIDs) == null ? void 0 : _b.value) || "");
22307
21342
  for (const advertiserId of advertiserIds) {
22308
21343
  for (const nodeName in fields) {
22309
- this.processNode({
21344
+ await this.processNode({
22310
21345
  nodeName,
22311
21346
  advertiserId,
22312
21347
  fields: fields[nodeName] || []
@@ -22321,8 +21356,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22321
21356
  * @param {string} options.advertiserId - Advertiser ID
22322
21357
  * @param {Array<string>} options.fields - Array of fields to fetch
22323
21358
  */
22324
- processNode({ nodeName, advertiserId, fields }) {
22325
- this.processTimeSeriesNode({
21359
+ async processNode({ nodeName, advertiserId, fields }) {
21360
+ await this.processTimeSeriesNode({
22326
21361
  nodeName,
22327
21362
  advertiserId,
22328
21363
  fields
@@ -22336,7 +21371,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22336
21371
  * @param {Array<string>} options.fields - Array of fields to fetch
22337
21372
  * @param {Object} options.storage - Storage instance
22338
21373
  */
22339
- processTimeSeriesNode({ nodeName, advertiserId, fields }) {
21374
+ async processTimeSeriesNode({ nodeName, advertiserId, fields }) {
22340
21375
  var _a;
22341
21376
  const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
22342
21377
  if (daysToFetch <= 0) {
@@ -22346,8 +21381,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22346
21381
  for (let i = 0; i < daysToFetch; i++) {
22347
21382
  const currentDate = new Date(startDate);
22348
21383
  currentDate.setDate(currentDate.getDate() + i);
22349
- const formattedDate = EnvironmentAdapter2.formatDate(currentDate, "UTC", "yyyy-MM-dd");
22350
- const data = this.source.fetchData({
21384
+ const formattedDate = DateUtils2.formatDate(currentDate);
21385
+ const data = await this.source.fetchData({
22351
21386
  nodeName,
22352
21387
  accountId: advertiserId,
22353
21388
  date: currentDate,
@@ -22356,7 +21391,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22356
21391
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${advertiserId} on ${formattedDate}` : `No records have been fetched`);
22357
21392
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
22358
21393
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
22359
- this.getStorageByNode(nodeName).saveData(preparedData);
21394
+ const storage = await this.getStorageByNode(nodeName);
21395
+ await storage.saveData(preparedData);
22360
21396
  }
22361
21397
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
22362
21398
  this.config.updateLastRequstedDate(currentDate);
@@ -22368,7 +21404,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22368
21404
  * @param {string} nodeName - Name of the node
22369
21405
  * @returns {Object} Storage instance
22370
21406
  */
22371
- getStorageByNode(nodeName) {
21407
+ async getStorageByNode(nodeName) {
22372
21408
  if (!("storages" in this)) {
22373
21409
  this.storages = {};
22374
21410
  }
@@ -22386,6 +21422,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22386
21422
  this.source.fieldsSchema[nodeName].fields,
22387
21423
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
22388
21424
  );
21425
+ await this.storages[nodeName].init();
22389
21426
  }
22390
21427
  return this.storages[nodeName];
22391
21428
  }
@@ -22402,7 +21439,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22402
21439
  };
22403
21440
  })();
22404
21441
  const BankOfCanada = (function() {
22405
- const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
21442
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
22406
21443
  var observationsFields = {
22407
21444
  "date": {
22408
21445
  "description": "The date for which the exchange rate was recorded.",
@@ -22480,10 +21517,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22480
21517
  * @param {string} [opts.end_time]
22481
21518
  * @returns {Array<Object>}
22482
21519
  */
22483
- fetchData({ nodeName, fields = [], start_time, end_time }) {
21520
+ async fetchData({ nodeName, fields = [], start_time, end_time }) {
22484
21521
  switch (nodeName) {
22485
21522
  case "observations/group":
22486
- return this._fetchObservations({ fields, start_time, end_time });
21523
+ return await this._fetchObservations({ fields, start_time, end_time });
22487
21524
  default:
22488
21525
  throw new Error(`Unknown node: ${nodeName}`);
22489
21526
  }
@@ -22496,8 +21533,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22496
21533
  * @param {string} opts.end_time
22497
21534
  * @returns {Array<Object>}
22498
21535
  */
22499
- _fetchObservations({ fields, start_time, end_time }) {
22500
- const rates = this.makeRequest({
21536
+ async _fetchObservations({ fields, start_time, end_time }) {
21537
+ const rates = await this.makeRequest({
22501
21538
  endpoint: `observations/group/FX_RATES_DAILY/json?start_date=${start_time}&end_date=${end_time}`
22502
21539
  });
22503
21540
  let data = [];
@@ -22519,13 +21556,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22519
21556
  * @param {string} options.endpoint - API endpoint path (e.g., "observations/group/FX_RATES_DAILY/json")
22520
21557
  * @returns {Object} - API response parsed from JSON
22521
21558
  */
22522
- makeRequest({ endpoint }) {
21559
+ async makeRequest({ endpoint }) {
22523
21560
  const baseUrl = "https://www.bankofcanada.ca/valet/";
22524
21561
  const url = `${baseUrl}${endpoint}`;
22525
21562
  console.log(`Bank of Canada API Request URL:`, url);
22526
- const response = EnvironmentAdapter2.fetch(url, { "method": "get", "muteHttpExceptions": true });
22527
- const result = JSON.parse(response.getContentText());
22528
- return result;
21563
+ const response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
21564
+ const result = await response.getContentText();
21565
+ return JSON.parse(result);
22529
21566
  }
22530
21567
  /**
22531
21568
  * Keep only requestedFields plus any schema-required keys.
@@ -22550,7 +21587,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22550
21587
  }
22551
21588
  };
22552
21589
  var BankOfCanadaConnector = class BankOfCanadaConnector extends AbstractConnector2 {
22553
- constructor(config, source, storageName = "GoogleSheetsStorage", runConfig = null) {
21590
+ constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
22554
21591
  super(config, source, null, runConfig);
22555
21592
  this.storageName = storageName;
22556
21593
  }
@@ -22558,10 +21595,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22558
21595
  * Main method - entry point for the import process
22559
21596
  * Processes all nodes defined in the fields configuration
22560
21597
  */
22561
- startImportProcess() {
21598
+ async startImportProcess() {
22562
21599
  const fields = ConnectorUtils.parseFields(this.config.Fields.value);
22563
21600
  for (const nodeName in fields) {
22564
- this.processNode({
21601
+ await this.processNode({
22565
21602
  nodeName,
22566
21603
  fields: fields[nodeName] || []
22567
21604
  });
@@ -22573,11 +21610,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22573
21610
  * @param {string} options.nodeName - Name of the node to process
22574
21611
  * @param {Array<string>} options.fields - Array of fields to fetch
22575
21612
  */
22576
- processNode({ nodeName, fields }) {
21613
+ async processNode({ nodeName, fields }) {
22577
21614
  if (this.source.fieldsSchema[nodeName].isTimeSeries) {
22578
- this.processTimeSeriesNode({ nodeName, fields });
21615
+ await this.processTimeSeriesNode({ nodeName, fields });
22579
21616
  } else {
22580
- this.processCatalogNode({ nodeName, fields });
21617
+ await this.processCatalogNode({ nodeName, fields });
22581
21618
  }
22582
21619
  }
22583
21620
  /**
@@ -22587,14 +21624,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22587
21624
  * @param {Array<string>} options.fields - Array of fields to fetch
22588
21625
  * @param {Object} options.storage - Storage instance
22589
21626
  */
22590
- processTimeSeriesNode({ nodeName, fields }) {
21627
+ async processTimeSeriesNode({ nodeName, fields }) {
22591
21628
  var _a;
22592
21629
  const dateRange = this.prepareDateRange();
22593
21630
  if (!dateRange) {
22594
21631
  console.log("No days to fetch for time series data");
22595
21632
  return;
22596
21633
  }
22597
- const data = this.source.fetchData({
21634
+ const data = await this.source.fetchData({
22598
21635
  nodeName,
22599
21636
  start_time: dateRange.startDate,
22600
21637
  end_time: dateRange.endDate,
@@ -22603,7 +21640,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22603
21640
  this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
22604
21641
  if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
22605
21642
  const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
22606
- this.getStorageByNode(nodeName).saveData(preparedData);
21643
+ const storage = await this.getStorageByNode(nodeName);
21644
+ await storage.saveData(preparedData);
22607
21645
  }
22608
21646
  if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
22609
21647
  this.config.updateLastRequstedDate(new Date(dateRange.endDate));
@@ -22616,7 +21654,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22616
21654
  * @param {Array<string>} options.fields - Array of fields to fetch
22617
21655
  * @param {Object} options.storage - Storage instance
22618
21656
  */
22619
- processCatalogNode({ nodeName, fields }) {
21657
+ async processCatalogNode({ nodeName, fields }) {
22620
21658
  console.log(`Catalog node processing not implemented for ${nodeName}`);
22621
21659
  }
22622
21660
  /**
@@ -22624,7 +21662,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22624
21662
  * @param {string} nodeName - Name of the node
22625
21663
  * @returns {Object} Storage instance
22626
21664
  */
22627
- getStorageByNode(nodeName) {
21665
+ async getStorageByNode(nodeName) {
22628
21666
  if (!("storages" in this)) {
22629
21667
  this.storages = {};
22630
21668
  }
@@ -22642,6 +21680,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22642
21680
  this.source.fieldsSchema[nodeName].fields,
22643
21681
  `${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
22644
21682
  );
21683
+ await this.storages[nodeName].init();
22645
21684
  }
22646
21685
  return this.storages[nodeName];
22647
21686
  }
@@ -22657,8 +21696,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22657
21696
  const endDate = new Date(startDate);
22658
21697
  endDate.setDate(endDate.getDate() + daysToFetch - 1);
22659
21698
  return {
22660
- startDate: EnvironmentAdapter2.formatDate(startDate, "UTC", "yyyy-MM-dd"),
22661
- endDate: EnvironmentAdapter2.formatDate(endDate, "UTC", "yyyy-MM-dd")
21699
+ startDate: DateUtils2.formatDate(startDate),
21700
+ endDate: DateUtils2.formatDate(endDate)
22662
21701
  };
22663
21702
  }
22664
21703
  };
@@ -22704,7 +21743,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22704
21743
  "BankOfCanada"
22705
21744
  ];
22706
21745
  const AvailableStorages = [
22707
- "GoogleSheets",
22708
21746
  "GoogleBigQuery",
22709
21747
  "AwsAthena"
22710
21748
  ];
@@ -22729,7 +21767,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
22729
21767
  CriteoAds,
22730
21768
  BankOfCanada,
22731
21769
  // Individual storages
22732
- GoogleSheets,
22733
21770
  GoogleBigQuery,
22734
21771
  AwsAthena
22735
21772
  };