@owox/connectors 0.12.0-next-20251105085230 → 0.12.0-next-20251105123137
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connector-runner.cjs +1 -5
- package/dist/connector-runner.js +1 -5
- package/dist/index.cjs +800 -1763
- package/dist/index.js +800 -1763
- package/package.json +1 -3
package/dist/index.cjs
CHANGED
|
@@ -20,11 +20,6 @@ var HTTP_STATUS = {
|
|
|
20
20
|
SERVER_ERROR_MIN: 500,
|
|
21
21
|
SERVER_ERROR_MAX: 599
|
|
22
22
|
};
|
|
23
|
-
var ENVIRONMENT = {
|
|
24
|
-
UNKNOWN: 1,
|
|
25
|
-
APPS_SCRIPT: 2,
|
|
26
|
-
NODE: 3
|
|
27
|
-
};
|
|
28
23
|
var EXECUTION_STATUS = {
|
|
29
24
|
IMPORT_IN_PROGRESS: 1,
|
|
30
25
|
CLEANUP_IN_PROGRESS: 2,
|
|
@@ -72,269 +67,11 @@ class HttpRequestException extends AbstractException {
|
|
|
72
67
|
}
|
|
73
68
|
class UnsupportedEnvironmentException extends AbstractException {
|
|
74
69
|
}
|
|
75
|
-
var EnvironmentAdapter = class EnvironmentAdapter2 {
|
|
76
|
-
/**
|
|
77
|
-
* Mac algorithm constants.
|
|
78
|
-
*
|
|
79
|
-
* @type {Object}
|
|
80
|
-
*/
|
|
81
|
-
static get MacAlgorithm() {
|
|
82
|
-
return {
|
|
83
|
-
HMAC_SHA_256: "HMAC_SHA_256",
|
|
84
|
-
HMAC_SHA_384: "HMAC_SHA_384",
|
|
85
|
-
HMAC_SHA_512: "HMAC_SHA_512",
|
|
86
|
-
HMAC_SHA_1: "HMAC_SHA_1",
|
|
87
|
-
HMAC_MD5: "HMAC_MD5"
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
constructor() {
|
|
91
|
-
this.environment = this.getEnvironment();
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* Get the current environment.
|
|
95
|
-
* Detects whether code is running in Google Apps Script or Node.js environment.
|
|
96
|
-
*
|
|
97
|
-
* @returns {ENVIRONMENT} The detected environment (APPS_SCRIPT, NODE, or UNKNOWN)
|
|
98
|
-
* @throws {UnsupportedEnvironmentException} If environment cannot be determined
|
|
99
|
-
*/
|
|
100
|
-
static getEnvironment() {
|
|
101
|
-
if (typeof this.environment !== "undefined") {
|
|
102
|
-
return this.environment;
|
|
103
|
-
}
|
|
104
|
-
if (typeof UrlFetchApp !== "undefined") {
|
|
105
|
-
this.environment = ENVIRONMENT.APPS_SCRIPT;
|
|
106
|
-
} else if (typeof process !== "undefined") {
|
|
107
|
-
this.environment = ENVIRONMENT.NODE;
|
|
108
|
-
} else {
|
|
109
|
-
this.environment = ENVIRONMENT.UNKNOWN;
|
|
110
|
-
}
|
|
111
|
-
return this.environment;
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Fetch data from the given URL.
|
|
115
|
-
*
|
|
116
|
-
* @param {string} url - The URL to fetch data from.
|
|
117
|
-
* @param {Object} options - Options for the fetch request.
|
|
118
|
-
* @returns {FetchResponse}
|
|
119
|
-
*
|
|
120
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
121
|
-
*/
|
|
122
|
-
static fetch(url, options = {}) {
|
|
123
|
-
const env = this.getEnvironment();
|
|
124
|
-
if (env === ENVIRONMENT.APPS_SCRIPT) {
|
|
125
|
-
const response = UrlFetchApp.fetch(url, options);
|
|
126
|
-
return this._wrapAppsScriptResponse(response);
|
|
127
|
-
}
|
|
128
|
-
if (env === ENVIRONMENT.NODE) {
|
|
129
|
-
const method = options.method || "GET";
|
|
130
|
-
const response = request(method, url, options);
|
|
131
|
-
return this._wrapNodeResponse(response);
|
|
132
|
-
}
|
|
133
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Sleep for the given number of milliseconds.
|
|
137
|
-
*
|
|
138
|
-
* @param {number} ms - The number of milliseconds to sleep.
|
|
139
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
140
|
-
*/
|
|
141
|
-
static sleep(ms) {
|
|
142
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
143
|
-
Utilities.sleep(ms);
|
|
144
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
145
|
-
let done = false;
|
|
146
|
-
new Promise((resolve) => {
|
|
147
|
-
setTimeout(() => {
|
|
148
|
-
done = true;
|
|
149
|
-
resolve();
|
|
150
|
-
}, ms);
|
|
151
|
-
});
|
|
152
|
-
deasync.loopWhile(() => !done);
|
|
153
|
-
} else {
|
|
154
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
/**
|
|
158
|
-
* Format the given date.
|
|
159
|
-
*
|
|
160
|
-
* @param {Date} date - The date to format.
|
|
161
|
-
* @param {string} timezone - The timezone to format the date in.
|
|
162
|
-
* @param {string} format - The format to format the date in.
|
|
163
|
-
* @returns {string}
|
|
164
|
-
*
|
|
165
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
166
|
-
*/
|
|
167
|
-
static formatDate(date, timezone, format) {
|
|
168
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
169
|
-
return Utilities.formatDate(date, timezone, format);
|
|
170
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
171
|
-
return date.toISOString().split("T")[0];
|
|
172
|
-
} else {
|
|
173
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
/**
|
|
177
|
-
* Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
|
|
178
|
-
*
|
|
179
|
-
* @returns {string} UUID
|
|
180
|
-
*
|
|
181
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
182
|
-
*/
|
|
183
|
-
static getUuid() {
|
|
184
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
185
|
-
return Utilities.getUuid();
|
|
186
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
187
|
-
const crypto = require("node:crypto");
|
|
188
|
-
return crypto.randomUUID();
|
|
189
|
-
} else {
|
|
190
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
/**
|
|
194
|
-
* Encode the given data to base64.
|
|
195
|
-
*
|
|
196
|
-
* @param {string} data - The data to encode.
|
|
197
|
-
* @returns {string}
|
|
198
|
-
*
|
|
199
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
200
|
-
*/
|
|
201
|
-
static base64Encode(data) {
|
|
202
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
203
|
-
return Utilities.base64Encode(data);
|
|
204
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
205
|
-
return Buffer.from(data).toString("base64");
|
|
206
|
-
} else {
|
|
207
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
/**
|
|
211
|
-
* Compute the HMAC signature for the given data.
|
|
212
|
-
*
|
|
213
|
-
* @param {string} algorithm - The algorithm to use.
|
|
214
|
-
* @param {string} data - The data to compute the signature for.
|
|
215
|
-
* @param {string} key - The key to use.
|
|
216
|
-
* @returns {string}
|
|
217
|
-
*
|
|
218
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
219
|
-
*/
|
|
220
|
-
static computeHmacSignature(algorithm, data, key) {
|
|
221
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
222
|
-
if (typeof algorithm === "string") {
|
|
223
|
-
algorithm = Utilities.MacAlgorithm[algorithm];
|
|
224
|
-
}
|
|
225
|
-
return Utilities.computeHmacSignature(algorithm, data, key);
|
|
226
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
227
|
-
const crypto = require("node:crypto");
|
|
228
|
-
const algorithmMap = {
|
|
229
|
-
"HMAC_SHA_256": "sha256",
|
|
230
|
-
"HMAC_SHA_384": "sha384",
|
|
231
|
-
"HMAC_SHA_512": "sha512",
|
|
232
|
-
"HMAC_SHA_1": "sha1",
|
|
233
|
-
"HMAC_MD5": "md5"
|
|
234
|
-
};
|
|
235
|
-
const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
|
|
236
|
-
const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
|
|
237
|
-
return Array.from(buffer);
|
|
238
|
-
} else {
|
|
239
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
/**
|
|
243
|
-
* Parse CSV string into array of arrays
|
|
244
|
-
*
|
|
245
|
-
* @param {string} csvString - The CSV string to parse
|
|
246
|
-
* @param {string} [delimiter=','] - The delimiter to use for parsing CSV
|
|
247
|
-
* @returns {Array<Array<string>>} Parsed CSV data
|
|
248
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported
|
|
249
|
-
*/
|
|
250
|
-
static parseCsv(csvString, delimiter = ",") {
|
|
251
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
252
|
-
return Utilities.parseCsv(csvString, delimiter);
|
|
253
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
254
|
-
return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
|
|
255
|
-
const trimmed = cell.trim();
|
|
256
|
-
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
257
|
-
return trimmed.slice(1, -1).replace(/""/g, '"');
|
|
258
|
-
}
|
|
259
|
-
return trimmed;
|
|
260
|
-
}));
|
|
261
|
-
} else {
|
|
262
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
/**
|
|
266
|
-
* Unzip a blob/buffer
|
|
267
|
-
*
|
|
268
|
-
* @param {Blob|Buffer} data - The data to unzip
|
|
269
|
-
* @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
|
|
270
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported
|
|
271
|
-
*/
|
|
272
|
-
static unzip(data) {
|
|
273
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
274
|
-
return Utilities.unzip(data);
|
|
275
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
276
|
-
const zip = new AdmZip(data);
|
|
277
|
-
return zip.getEntries().map((entry) => ({
|
|
278
|
-
getDataAsString: () => entry.getData().toString("utf8")
|
|
279
|
-
}));
|
|
280
|
-
} else {
|
|
281
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
/**
|
|
285
|
-
* Wraps the response from the Apps Script environment.
|
|
286
|
-
* Not use directly, only for internal purposes.
|
|
287
|
-
*
|
|
288
|
-
* @param {Object} response
|
|
289
|
-
* @returns {FetchResponse}
|
|
290
|
-
*/
|
|
291
|
-
static _wrapAppsScriptResponse(response) {
|
|
292
|
-
return {
|
|
293
|
-
getHeaders: () => response.getAllHeaders(),
|
|
294
|
-
getAsJson: () => {
|
|
295
|
-
try {
|
|
296
|
-
return JSON.parse(response.getContentText());
|
|
297
|
-
} catch (e) {
|
|
298
|
-
throw new Error("Invalid JSON response");
|
|
299
|
-
}
|
|
300
|
-
},
|
|
301
|
-
getContent: () => response.getContent(),
|
|
302
|
-
getContentText: () => response.getContentText(),
|
|
303
|
-
getBlob: () => response.getBlob(),
|
|
304
|
-
getResponseCode: () => response.getResponseCode()
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
/**
|
|
308
|
-
* Wraps the response from the Node environment.
|
|
309
|
-
* Not use directly, only for internal purposes.
|
|
310
|
-
*
|
|
311
|
-
* @param {Object} response
|
|
312
|
-
* @returns {FetchResponse}
|
|
313
|
-
*/
|
|
314
|
-
static _wrapNodeResponse(response) {
|
|
315
|
-
const headers = response.headers || {};
|
|
316
|
-
const text = response.body ? response.body.toString() : "";
|
|
317
|
-
return {
|
|
318
|
-
getHeaders: () => headers,
|
|
319
|
-
getAsJson: () => {
|
|
320
|
-
try {
|
|
321
|
-
return JSON.parse(text);
|
|
322
|
-
} catch (e) {
|
|
323
|
-
throw new Error("Invalid JSON response");
|
|
324
|
-
}
|
|
325
|
-
},
|
|
326
|
-
getContent: () => text,
|
|
327
|
-
getContentText: () => text,
|
|
328
|
-
getBlob: () => response.body,
|
|
329
|
-
getResponseCode: () => response.statusCode
|
|
330
|
-
};
|
|
331
|
-
}
|
|
332
|
-
};
|
|
333
70
|
class AbstractStorage {
|
|
334
71
|
//---- constructor -------------------------------------------------
|
|
335
72
|
/**
|
|
336
|
-
*
|
|
337
|
-
* @param config (object) instance of
|
|
73
|
+
* Abstract class for storage operations providing common methods for data persistence
|
|
74
|
+
* @param config (object) instance of AbstractConfig
|
|
338
75
|
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
339
76
|
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
340
77
|
* @param description (string) string with storage description }
|
|
@@ -358,6 +95,14 @@ class AbstractStorage {
|
|
|
358
95
|
}
|
|
359
96
|
}
|
|
360
97
|
//----------------------------------------------------------------
|
|
98
|
+
//---- init --------------------------------------------------------
|
|
99
|
+
/**
|
|
100
|
+
* Initializing storage
|
|
101
|
+
*/
|
|
102
|
+
async init() {
|
|
103
|
+
throw new Error("Method init() has to be implemented in a child class of AbstractStorage");
|
|
104
|
+
}
|
|
105
|
+
//----------------------------------------------------------------
|
|
361
106
|
//---- getUniqueKeyByRecordFields ----------------------------------
|
|
362
107
|
/**
|
|
363
108
|
* Calculcating unique key based on this.uniqueKeyColumns
|
|
@@ -414,11 +159,12 @@ class AbstractStorage {
|
|
|
414
159
|
//----------------------------------------------------------------
|
|
415
160
|
//---- saveData ----------------------------------------------------
|
|
416
161
|
/**
|
|
417
|
-
* Saving data to a storage. Has to be implemented in
|
|
162
|
+
* Saving data to a storage. Has to be implemented in child class as async method.
|
|
418
163
|
* @param {data} array of assoc objects with records to save
|
|
164
|
+
* @returns {Promise<void>}
|
|
419
165
|
*/
|
|
420
|
-
saveData(data) {
|
|
421
|
-
throw new Error("Method
|
|
166
|
+
async saveData(data) {
|
|
167
|
+
throw new Error("Method saveData() has to be implemented in a child class of AbstractStorage");
|
|
422
168
|
}
|
|
423
169
|
//----------------------------------------------------------------
|
|
424
170
|
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
@@ -489,16 +235,6 @@ class AbstractStorage {
|
|
|
489
235
|
return record;
|
|
490
236
|
}
|
|
491
237
|
//----------------------------------------------------------------
|
|
492
|
-
//---- areHeadersNeeded --------------------------------------------
|
|
493
|
-
/**
|
|
494
|
-
* Checks if storage needs headers to be added
|
|
495
|
-
* By default returns false, should be overridden in child classes if needed
|
|
496
|
-
* @returns {boolean} true if headers need to be added, false otherwise
|
|
497
|
-
*/
|
|
498
|
-
areHeadersNeeded() {
|
|
499
|
-
return false;
|
|
500
|
-
}
|
|
501
|
-
//----------------------------------------------------------------
|
|
502
238
|
//---- getSelectedFields -------------------------------------------
|
|
503
239
|
/**
|
|
504
240
|
* Parse Fields config value and return array of selected field names
|
|
@@ -552,7 +288,7 @@ var AbstractSource = class AbstractSource2 {
|
|
|
552
288
|
* A Data Source-specific methid is used to fetch new data and return it as an array of objects, where each property of an object corresponds to a column name.
|
|
553
289
|
* @return data array
|
|
554
290
|
*/
|
|
555
|
-
fetchData() {
|
|
291
|
+
async fetchData() {
|
|
556
292
|
throw new Error("Method fetchData must be implemented in Class inheritor of AbstractSource");
|
|
557
293
|
}
|
|
558
294
|
//----------------------------------------------------------------
|
|
@@ -577,16 +313,16 @@ var AbstractSource = class AbstractSource2 {
|
|
|
577
313
|
* @return {HTTPResponse} The response object from the fetch
|
|
578
314
|
* @throws {HttpRequestException} After exhausting all retries
|
|
579
315
|
*/
|
|
580
|
-
urlFetchWithRetry(url, options) {
|
|
316
|
+
async urlFetchWithRetry(url, options) {
|
|
581
317
|
for (let attempt = 1; attempt <= this.config.MaxFetchRetries.value; attempt++) {
|
|
582
318
|
try {
|
|
583
|
-
const response =
|
|
584
|
-
return this._validateResponse(response);
|
|
319
|
+
const response = await HttpUtils.fetch(url, { ...options, muteHttpExceptions: true });
|
|
320
|
+
return await this._validateResponse(response);
|
|
585
321
|
} catch (error) {
|
|
586
322
|
if (!this._shouldRetry(error, attempt)) {
|
|
587
323
|
throw error;
|
|
588
324
|
}
|
|
589
|
-
this._waitBeforeRetry(attempt);
|
|
325
|
+
await this._waitBeforeRetry(attempt);
|
|
590
326
|
}
|
|
591
327
|
}
|
|
592
328
|
}
|
|
@@ -598,12 +334,12 @@ var AbstractSource = class AbstractSource2 {
|
|
|
598
334
|
* @throws {HttpRequestException} If the response indicates an error
|
|
599
335
|
* @private
|
|
600
336
|
*/
|
|
601
|
-
_validateResponse(response) {
|
|
337
|
+
async _validateResponse(response) {
|
|
602
338
|
const code = response.getResponseCode();
|
|
603
339
|
if (code >= HTTP_STATUS.SUCCESS_MIN && code <= HTTP_STATUS.SUCCESS_MAX) {
|
|
604
340
|
return response;
|
|
605
341
|
}
|
|
606
|
-
const errorInfo = this._extractErrorInfo(response);
|
|
342
|
+
const errorInfo = await this._extractErrorInfo(response);
|
|
607
343
|
throw new HttpRequestException({
|
|
608
344
|
message: errorInfo.message,
|
|
609
345
|
statusCode: code,
|
|
@@ -617,9 +353,9 @@ var AbstractSource = class AbstractSource2 {
|
|
|
617
353
|
* @return {Object} Object containing error message and JSON data if available
|
|
618
354
|
* @private
|
|
619
355
|
*/
|
|
620
|
-
_extractErrorInfo(response) {
|
|
356
|
+
async _extractErrorInfo(response) {
|
|
621
357
|
var _a, _b;
|
|
622
|
-
const text = response.getContentText();
|
|
358
|
+
const text = await response.getContentText();
|
|
623
359
|
let parsedJson = null;
|
|
624
360
|
let message = text;
|
|
625
361
|
try {
|
|
@@ -656,10 +392,10 @@ var AbstractSource = class AbstractSource2 {
|
|
|
656
392
|
* @param {number} attempt - The current attempt number
|
|
657
393
|
* @private
|
|
658
394
|
*/
|
|
659
|
-
_waitBeforeRetry(attempt) {
|
|
395
|
+
async _waitBeforeRetry(attempt) {
|
|
660
396
|
const delay = this.calculateBackoff(attempt);
|
|
661
397
|
console.log(`Retrying after ${Math.round(delay / 1e3)}s...`);
|
|
662
|
-
|
|
398
|
+
await AsyncUtils.delay(delay);
|
|
663
399
|
}
|
|
664
400
|
//---- calculateBackoff --------------------------------------------
|
|
665
401
|
/**
|
|
@@ -791,7 +527,7 @@ var AbstractConnector = class AbstractConnector2 {
|
|
|
791
527
|
/**
|
|
792
528
|
* Initiates imports new data from a data source
|
|
793
529
|
*/
|
|
794
|
-
run() {
|
|
530
|
+
async run() {
|
|
795
531
|
try {
|
|
796
532
|
if (this.config.isInProgress()) {
|
|
797
533
|
this.config.logMessage("Import is already in progress");
|
|
@@ -801,11 +537,7 @@ var AbstractConnector = class AbstractConnector2 {
|
|
|
801
537
|
this.config.handleStatusUpdate({ status: EXECUTION_STATUS.IMPORT_IN_PROGRESS });
|
|
802
538
|
this.config.updateLastImportDate();
|
|
803
539
|
this.config.logMessage("Start importing new data");
|
|
804
|
-
|
|
805
|
-
this.storage.addHeader(this.storage.uniqueKeyColumns);
|
|
806
|
-
this.config.logMessage(`Column(s) for unique key was added: ${this.storage.uniqueKeyColumns}`);
|
|
807
|
-
}
|
|
808
|
-
this.startImportProcess();
|
|
540
|
+
await this.startImportProcess();
|
|
809
541
|
this.config.logMessage("Import is finished");
|
|
810
542
|
this.config.handleStatusUpdate({
|
|
811
543
|
status: EXECUTION_STATUS.IMPORT_DONE
|
|
@@ -826,7 +558,7 @@ var AbstractConnector = class AbstractConnector2 {
|
|
|
826
558
|
/**
|
|
827
559
|
* A method for calling from Root script for determining parameters needed to fetch new data.
|
|
828
560
|
*/
|
|
829
|
-
startImportProcess() {
|
|
561
|
+
async startImportProcess() {
|
|
830
562
|
var _a;
|
|
831
563
|
let startDate = null;
|
|
832
564
|
let endDate = /* @__PURE__ */ new Date();
|
|
@@ -837,10 +569,10 @@ var AbstractConnector = class AbstractConnector2 {
|
|
|
837
569
|
return;
|
|
838
570
|
}
|
|
839
571
|
endDate.setDate(startDate.getDate() + daysToFetch);
|
|
840
|
-
let data = this.source.fetchData(startDate, endDate);
|
|
572
|
+
let data = await this.source.fetchData(startDate, endDate);
|
|
841
573
|
this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
|
|
842
574
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value) === "true") {
|
|
843
|
-
this.storage.saveData(data);
|
|
575
|
+
await this.storage.saveData(data);
|
|
844
576
|
}
|
|
845
577
|
if (this.runConfig.type === RUN_CONFIG_TYPE.INCREMENTAL) {
|
|
846
578
|
this.config.updateLastRequstedDate(endDate);
|
|
@@ -997,31 +729,12 @@ class AbstractConfig {
|
|
|
997
729
|
* @param (object) with config data. Properties are parameters names, values are values
|
|
998
730
|
*/
|
|
999
731
|
constructor(configData) {
|
|
1000
|
-
this.addParameter("Environment", {
|
|
1001
|
-
value: AbstractConfig.detectEnvironment(),
|
|
1002
|
-
requiredType: "number",
|
|
1003
|
-
attributes: [CONFIG_ATTRIBUTES.HIDE_IN_CONFIG_FORM, CONFIG_ATTRIBUTES.ADVANCED]
|
|
1004
|
-
});
|
|
1005
732
|
for (var name in configData) {
|
|
1006
733
|
this.addParameter(name, configData[name]);
|
|
1007
734
|
}
|
|
1008
735
|
return this;
|
|
1009
736
|
}
|
|
1010
737
|
//----------------------------------------------------------------
|
|
1011
|
-
//---- static helper -------------------------------------------------
|
|
1012
|
-
/**
|
|
1013
|
-
* Determines the runtime environment
|
|
1014
|
-
* @returns {ENVIRONMENT} The detected environment
|
|
1015
|
-
*/
|
|
1016
|
-
static detectEnvironment() {
|
|
1017
|
-
if (typeof UrlFetchApp !== "undefined") {
|
|
1018
|
-
return ENVIRONMENT.APPS_SCRIPT;
|
|
1019
|
-
}
|
|
1020
|
-
if (typeof process !== "undefined") {
|
|
1021
|
-
return ENVIRONMENT.NODE;
|
|
1022
|
-
}
|
|
1023
|
-
return ENVIRONMENT.UNKNOWN;
|
|
1024
|
-
}
|
|
1025
738
|
//---- mergeParameters ---------------------------------------------
|
|
1026
739
|
/**
|
|
1027
740
|
* Merge configuration to existing config
|
|
@@ -1228,11 +941,11 @@ class AbstractConfig {
|
|
|
1228
941
|
}
|
|
1229
942
|
//----------------------------------------------------------------
|
|
1230
943
|
}
|
|
1231
|
-
function processShortLinks(data, { shortLinkField, urlFieldName }) {
|
|
944
|
+
async function processShortLinks(data, { shortLinkField, urlFieldName }) {
|
|
1232
945
|
if (!Array.isArray(data) || data.length === 0) return data;
|
|
1233
946
|
const shortLinks = _collectUniqueShortLinks(data, shortLinkField, urlFieldName);
|
|
1234
947
|
if (shortLinks.length === 0) return data;
|
|
1235
|
-
const resolvedShortLinks = _resolveShortLinks(shortLinks);
|
|
948
|
+
const resolvedShortLinks = await _resolveShortLinks(shortLinks);
|
|
1236
949
|
return _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName);
|
|
1237
950
|
}
|
|
1238
951
|
function _collectUniqueShortLinks(data, shortLinkField, urlFieldName) {
|
|
@@ -1254,13 +967,11 @@ function _isPotentialShortLink(url) {
|
|
|
1254
967
|
if (hasParams) return false;
|
|
1255
968
|
return /^https:\/\/[^\/]+\/[^\/]+$/.test(url);
|
|
1256
969
|
}
|
|
1257
|
-
function _resolveShortLinks(shortLinks) {
|
|
1258
|
-
|
|
970
|
+
async function _resolveShortLinks(shortLinks) {
|
|
971
|
+
const promises = shortLinks.map(async (linkObj) => {
|
|
1259
972
|
try {
|
|
1260
|
-
const response =
|
|
1261
|
-
method: "GET"
|
|
1262
|
-
followRedirects: false,
|
|
1263
|
-
muteHttpExceptions: true
|
|
973
|
+
const response = await HttpUtils.fetch(linkObj.originalUrl, {
|
|
974
|
+
method: "GET"
|
|
1264
975
|
});
|
|
1265
976
|
const headers = response.getHeaders();
|
|
1266
977
|
const resolvedUrl = headers.Location || headers.location || linkObj.originalUrl;
|
|
@@ -1276,6 +987,7 @@ function _resolveShortLinks(shortLinks) {
|
|
|
1276
987
|
};
|
|
1277
988
|
}
|
|
1278
989
|
});
|
|
990
|
+
return Promise.all(promises);
|
|
1279
991
|
}
|
|
1280
992
|
function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName) {
|
|
1281
993
|
return data.map((record) => {
|
|
@@ -1299,15 +1011,15 @@ function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField,
|
|
|
1299
1011
|
var OAuthUtils = {
|
|
1300
1012
|
/**
|
|
1301
1013
|
* Universal OAuth access token retrieval method
|
|
1302
|
-
*
|
|
1014
|
+
*
|
|
1303
1015
|
* @param {Object} options - All configuration options
|
|
1304
1016
|
* @param {Object} options.config - Configuration object containing credentials
|
|
1305
1017
|
* @param {string} options.tokenUrl - OAuth token endpoint URL
|
|
1306
1018
|
* @param {Object} options.formData - Form data to send in request body
|
|
1307
1019
|
* @param {Object} [options.headers] - Request headers
|
|
1308
|
-
* @returns {string} - The access token
|
|
1020
|
+
* @returns {Promise<string>} - The access token
|
|
1309
1021
|
*/
|
|
1310
|
-
getAccessToken({ config, tokenUrl, formData, headers = {} }) {
|
|
1022
|
+
async getAccessToken({ config, tokenUrl, formData, headers = {} }) {
|
|
1311
1023
|
const requestHeaders = {
|
|
1312
1024
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
1313
1025
|
...headers
|
|
@@ -1320,8 +1032,9 @@ var OAuthUtils = {
|
|
|
1320
1032
|
body: Object.entries(formData).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
|
|
1321
1033
|
};
|
|
1322
1034
|
try {
|
|
1323
|
-
const resp =
|
|
1324
|
-
const
|
|
1035
|
+
const resp = await HttpUtils.fetch(tokenUrl, options);
|
|
1036
|
+
const text = await resp.getContentText();
|
|
1037
|
+
const json = JSON.parse(text);
|
|
1325
1038
|
if (json.error) {
|
|
1326
1039
|
throw new Error(`Token error: ${json.error}`);
|
|
1327
1040
|
}
|
|
@@ -1334,7 +1047,7 @@ var OAuthUtils = {
|
|
|
1334
1047
|
},
|
|
1335
1048
|
/**
|
|
1336
1049
|
* Get access token using Service Account JWT authentication
|
|
1337
|
-
*
|
|
1050
|
+
*
|
|
1338
1051
|
* @param {Object} options - Configuration options
|
|
1339
1052
|
* @param {Object} options.config - Configuration object
|
|
1340
1053
|
* @param {string} options.tokenUrl - Token URL
|
|
@@ -1342,7 +1055,7 @@ var OAuthUtils = {
|
|
|
1342
1055
|
* @param {string} options.scope - OAuth scope (e.g., "https://www.googleapis.com/auth/adwords")
|
|
1343
1056
|
* @returns {string} - The access token
|
|
1344
1057
|
*/
|
|
1345
|
-
getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
|
|
1058
|
+
async getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
|
|
1346
1059
|
try {
|
|
1347
1060
|
const serviceAccountData = JSON.parse(serviceAccountKeyJson);
|
|
1348
1061
|
const now = Math.floor(Date.now() / 1e3);
|
|
@@ -1360,7 +1073,7 @@ var OAuthUtils = {
|
|
|
1360
1073
|
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
1361
1074
|
assertion: jwt
|
|
1362
1075
|
};
|
|
1363
|
-
const accessToken = this.getAccessToken({
|
|
1076
|
+
const accessToken = await this.getAccessToken({
|
|
1364
1077
|
config,
|
|
1365
1078
|
tokenUrl,
|
|
1366
1079
|
formData
|
|
@@ -1413,6 +1126,68 @@ var OAuthUtils = {
|
|
|
1413
1126
|
return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
|
|
1414
1127
|
}
|
|
1415
1128
|
};
|
|
1129
|
+
var HttpUtils = class HttpUtils2 {
|
|
1130
|
+
/**
|
|
1131
|
+
* Fetch data from the given URL.
|
|
1132
|
+
*
|
|
1133
|
+
* @param {string} url - The URL to fetch data from.
|
|
1134
|
+
* @param {Object} options - Options for the fetch request.
|
|
1135
|
+
* @returns {Promise<FetchResponse>}
|
|
1136
|
+
*/
|
|
1137
|
+
static async fetch(url, options = {}) {
|
|
1138
|
+
const method = options.method || "GET";
|
|
1139
|
+
const fetchOptions = {
|
|
1140
|
+
method: method.toUpperCase(),
|
|
1141
|
+
headers: options.headers || {}
|
|
1142
|
+
};
|
|
1143
|
+
if (options.body) {
|
|
1144
|
+
fetchOptions.body = options.body;
|
|
1145
|
+
}
|
|
1146
|
+
const response = await fetch(url, fetchOptions);
|
|
1147
|
+
return this._wrapNodeResponse(response);
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* Wraps the response from the Node environment.
|
|
1151
|
+
* Not use directly, only for internal purposes.
|
|
1152
|
+
*
|
|
1153
|
+
* @param {Response} response - Native fetch Response object
|
|
1154
|
+
* @returns {FetchResponse}
|
|
1155
|
+
*/
|
|
1156
|
+
static _wrapNodeResponse(response) {
|
|
1157
|
+
let textCache = null;
|
|
1158
|
+
let blobCache = null;
|
|
1159
|
+
const getText = async () => {
|
|
1160
|
+
if (textCache === null) {
|
|
1161
|
+
textCache = await response.text();
|
|
1162
|
+
}
|
|
1163
|
+
return textCache;
|
|
1164
|
+
};
|
|
1165
|
+
const getBlob = async () => {
|
|
1166
|
+
if (blobCache === null) {
|
|
1167
|
+
blobCache = await response.arrayBuffer();
|
|
1168
|
+
}
|
|
1169
|
+
return Buffer.from(blobCache);
|
|
1170
|
+
};
|
|
1171
|
+
const headersObj = {};
|
|
1172
|
+
response.headers.forEach((value, key) => {
|
|
1173
|
+
headersObj[key] = value;
|
|
1174
|
+
});
|
|
1175
|
+
return {
|
|
1176
|
+
getHeaders: () => headersObj,
|
|
1177
|
+
getAsJson: () => getText().then((text) => {
|
|
1178
|
+
try {
|
|
1179
|
+
return JSON.parse(text);
|
|
1180
|
+
} catch (e) {
|
|
1181
|
+
throw new Error("Invalid JSON response");
|
|
1182
|
+
}
|
|
1183
|
+
}),
|
|
1184
|
+
getContent: () => getText(),
|
|
1185
|
+
getContentText: () => getText(),
|
|
1186
|
+
getBlob: () => getBlob(),
|
|
1187
|
+
getResponseCode: () => response.status
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
};
|
|
1416
1191
|
var FormatUtils = {
|
|
1417
1192
|
/**
|
|
1418
1193
|
* Universal ID parser: parses comma/semicolon separated string to array of numeric IDs
|
|
@@ -1455,6 +1230,102 @@ var FormatUtils = {
|
|
|
1455
1230
|
}, {});
|
|
1456
1231
|
}
|
|
1457
1232
|
};
|
|
1233
|
+
var FileUtils = class FileUtils2 {
|
|
1234
|
+
/**
|
|
1235
|
+
* Parse CSV string into array of arrays
|
|
1236
|
+
*
|
|
1237
|
+
* @param {string} csvString - The CSV string to parse
|
|
1238
|
+
* @param {string} [delimiter=','] - The delimiter to use for parsing CSV
|
|
1239
|
+
* @returns {Array<Array<string>>} Parsed CSV data
|
|
1240
|
+
*/
|
|
1241
|
+
static parseCsv(csvString, delimiter = ",") {
|
|
1242
|
+
return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
|
|
1243
|
+
const trimmed = cell.trim();
|
|
1244
|
+
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
1245
|
+
return trimmed.slice(1, -1).replace(/""/g, '"');
|
|
1246
|
+
}
|
|
1247
|
+
return trimmed;
|
|
1248
|
+
}));
|
|
1249
|
+
}
|
|
1250
|
+
/**
|
|
1251
|
+
* Unzip a blob/buffer
|
|
1252
|
+
*
|
|
1253
|
+
* @param {Buffer} data - The data to unzip
|
|
1254
|
+
* @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
|
|
1255
|
+
*/
|
|
1256
|
+
static unzip(data) {
|
|
1257
|
+
const zip = new AdmZip(data);
|
|
1258
|
+
return zip.getEntries().map((entry) => ({
|
|
1259
|
+
getDataAsString: () => entry.getData().toString("utf8")
|
|
1260
|
+
}));
|
|
1261
|
+
}
|
|
1262
|
+
};
|
|
1263
|
+
var DateUtils = class DateUtils2 {
|
|
1264
|
+
/**
|
|
1265
|
+
* Format the given date to ISO format (YYYY-MM-DD).
|
|
1266
|
+
*
|
|
1267
|
+
* @param {Date} date - The date to format.
|
|
1268
|
+
* @returns {string} ISO formatted date (YYYY-MM-DD)
|
|
1269
|
+
*/
|
|
1270
|
+
static formatDate(date) {
|
|
1271
|
+
return date.toISOString().split("T")[0];
|
|
1272
|
+
}
|
|
1273
|
+
};
|
|
1274
|
+
var CryptoUtils = class CryptoUtils2 {
|
|
1275
|
+
/**
|
|
1276
|
+
* Mac algorithm constants.
|
|
1277
|
+
*
|
|
1278
|
+
* @type {Object}
|
|
1279
|
+
*/
|
|
1280
|
+
static get MacAlgorithm() {
|
|
1281
|
+
return {
|
|
1282
|
+
HMAC_SHA_256: "HMAC_SHA_256",
|
|
1283
|
+
HMAC_SHA_384: "HMAC_SHA_384",
|
|
1284
|
+
HMAC_SHA_512: "HMAC_SHA_512",
|
|
1285
|
+
HMAC_SHA_1: "HMAC_SHA_1",
|
|
1286
|
+
HMAC_MD5: "HMAC_MD5"
|
|
1287
|
+
};
|
|
1288
|
+
}
|
|
1289
|
+
/**
|
|
1290
|
+
* Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
|
|
1291
|
+
*
|
|
1292
|
+
* @returns {string} UUID
|
|
1293
|
+
*/
|
|
1294
|
+
static getUuid() {
|
|
1295
|
+
const crypto = require("node:crypto");
|
|
1296
|
+
return crypto.randomUUID();
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Encode the given data to base64.
|
|
1300
|
+
*
|
|
1301
|
+
* @param {string} data - The data to encode.
|
|
1302
|
+
* @returns {string}
|
|
1303
|
+
*/
|
|
1304
|
+
static base64Encode(data) {
|
|
1305
|
+
return Buffer.from(data).toString("base64");
|
|
1306
|
+
}
|
|
1307
|
+
/**
|
|
1308
|
+
* Compute the HMAC signature for the given data.
|
|
1309
|
+
*
|
|
1310
|
+
* @param {string} algorithm - The algorithm to use (e.g., 'HMAC_SHA_256', 'sha256').
|
|
1311
|
+
* @param {string} data - The data to compute the signature for.
|
|
1312
|
+
* @param {string} key - The key to use.
|
|
1313
|
+
* @returns {Array<number>} Byte array of the signature
|
|
1314
|
+
*/
|
|
1315
|
+
static computeHmacSignature(algorithm, data, key) {
|
|
1316
|
+
const crypto = require("node:crypto");
|
|
1317
|
+
const algorithmMap = {
|
|
1318
|
+
"HMAC_SHA_256": "sha256",
|
|
1319
|
+
"HMAC_SHA_384": "sha384",
|
|
1320
|
+
"HMAC_SHA_512": "sha512",
|
|
1321
|
+
"HMAC_SHA_1": "sha1",
|
|
1322
|
+
"HMAC_MD5": "md5"
|
|
1323
|
+
};
|
|
1324
|
+
const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
|
|
1325
|
+
const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
|
|
1326
|
+
return Array.from(buffer);
|
|
1327
|
+
}
|
|
1328
|
+
};
|
|
1458
1329
|
var ConnectorUtils = {
|
|
1459
1330
|
/**
|
|
1460
1331
|
* Check if a node is a time series node
|
|
@@ -1477,6 +1348,17 @@ var ConnectorUtils = {
|
|
|
1477
1348
|
}, {});
|
|
1478
1349
|
}
|
|
1479
1350
|
};
|
|
1351
|
+
var AsyncUtils = class AsyncUtils2 {
|
|
1352
|
+
/**
|
|
1353
|
+
* Async delay for the given number of milliseconds.
|
|
1354
|
+
*
|
|
1355
|
+
* @param {number} ms - The number of milliseconds to delay.
|
|
1356
|
+
* @returns {Promise<void>}
|
|
1357
|
+
*/
|
|
1358
|
+
static async delay(ms) {
|
|
1359
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1360
|
+
}
|
|
1361
|
+
};
|
|
1480
1362
|
class RunConfigDto {
|
|
1481
1363
|
constructor(config) {
|
|
1482
1364
|
this._config = config;
|
|
@@ -1888,902 +1770,38 @@ class NodeJsConfig extends AbstractConfig {
|
|
|
1888
1770
|
);
|
|
1889
1771
|
}
|
|
1890
1772
|
}
|
|
1891
|
-
var GoogleSheetsConfig = class GoogleSheetsConfig2 extends AbstractConfig {
|
|
1892
|
-
//---- constructor -------------------------------------------------
|
|
1893
|
-
constructor(configRange) {
|
|
1894
|
-
if (typeof configRange.getA1Notation !== "function") {
|
|
1895
|
-
throw new Error(`Unable to create an GoogleSheetsConfig object. The first constructor's parameter must be an SpreadsheetApp.Sheet object`);
|
|
1896
|
-
}
|
|
1897
|
-
let configObject = {};
|
|
1898
|
-
configRange.getValues().forEach((row, index) => {
|
|
1899
|
-
var name = row[0].replaceAll(/[^a-zA-Z0-9]/gi, "");
|
|
1900
|
-
if (!["", "Parameters", "Status", "Name"].includes(name)) {
|
|
1901
|
-
var value = row[1];
|
|
1902
|
-
configObject[name] = {};
|
|
1903
|
-
if (value || value === 0) {
|
|
1904
|
-
configObject[name].value = value;
|
|
1905
|
-
}
|
|
1906
|
-
if (row[0].slice(-1) == "*") {
|
|
1907
|
-
configObject[name].isRequired = true;
|
|
1908
|
-
}
|
|
1909
|
-
if (["Log", "CurrentStatus", "LastImportDate", "LastRequestedDate", "DestinationSpreadsheet"].includes(name)) {
|
|
1910
|
-
configObject[name].cell = configRange.offset(index, 1, 1, 1);
|
|
1911
|
-
}
|
|
1912
|
-
}
|
|
1913
|
-
});
|
|
1914
|
-
configObject.configSpreadsheet = configRange.getSheet().getParent();
|
|
1915
|
-
configObject.Log.timeZone = configRange.getSheet().getParent().getSpreadsheetTimeZone();
|
|
1916
|
-
super(configObject);
|
|
1917
|
-
this.mergeParameters({
|
|
1918
|
-
MaxRunTimeout: {
|
|
1919
|
-
isRequired: true,
|
|
1920
|
-
requiredType: "number",
|
|
1921
|
-
default: 30
|
|
1922
|
-
},
|
|
1923
|
-
NotifyByEmail: {
|
|
1924
|
-
isRequired: false,
|
|
1925
|
-
requiredType: "string",
|
|
1926
|
-
default: ""
|
|
1927
|
-
},
|
|
1928
|
-
NotifyByGoogleChat: {
|
|
1929
|
-
isRequired: false,
|
|
1930
|
-
requiredType: "string",
|
|
1931
|
-
default: ""
|
|
1932
|
-
},
|
|
1933
|
-
NotifyWhen: {
|
|
1934
|
-
isRequired: false,
|
|
1935
|
-
requiredType: "string",
|
|
1936
|
-
default: "Never"
|
|
1937
|
-
}
|
|
1938
|
-
});
|
|
1939
|
-
}
|
|
1940
|
-
//---- handleStatusUpdate -----------------------------------------------
|
|
1941
|
-
/**
|
|
1942
|
-
* @param {Object} params - Parameters object with status and other properties
|
|
1943
|
-
* @param {number} params.status - Status constant
|
|
1944
|
-
* @param {string} params.error - Error message for Error status
|
|
1945
|
-
*/
|
|
1946
|
-
handleStatusUpdate({ status, error }) {
|
|
1947
|
-
this.manageTimeoutTrigger(status);
|
|
1948
|
-
this.updateCurrentStatus(status);
|
|
1949
|
-
if (this.shouldSendNotifications(status)) {
|
|
1950
|
-
this.sendNotifications({ status, error });
|
|
1951
|
-
}
|
|
1952
|
-
}
|
|
1953
|
-
//----------------------------------------------------------------
|
|
1954
|
-
//---- manageTimeoutTrigger ----------------------------------------
|
|
1955
|
-
/**
|
|
1956
|
-
* Manage timeout trigger based on current status
|
|
1957
|
-
* @param {number} status - Status constant
|
|
1958
|
-
*/
|
|
1959
|
-
manageTimeoutTrigger(status) {
|
|
1960
|
-
if (status === EXECUTION_STATUS.IMPORT_IN_PROGRESS) {
|
|
1961
|
-
this.createTimeoutTrigger();
|
|
1962
|
-
} else if (status === EXECUTION_STATUS.IMPORT_DONE || status === EXECUTION_STATUS.ERROR) {
|
|
1963
|
-
this.removeTimeoutTrigger();
|
|
1964
|
-
}
|
|
1965
|
-
}
|
|
1966
|
-
//---- getStatusProperties ------------------------------------------
|
|
1967
|
-
/**
|
|
1968
|
-
* Get all properties for a given status
|
|
1969
|
-
* @param {number} status - Status constant
|
|
1970
|
-
* @returns {Object} - Object with all status properties
|
|
1971
|
-
*/
|
|
1972
|
-
getStatusProperties(status) {
|
|
1973
|
-
switch (status) {
|
|
1974
|
-
case EXECUTION_STATUS.IMPORT_IN_PROGRESS:
|
|
1975
|
-
return {
|
|
1976
|
-
displayText: "Import in progress",
|
|
1977
|
-
backgroundColor: "#c9e3f9",
|
|
1978
|
-
notificationMessage: "Import is in progress."
|
|
1979
|
-
};
|
|
1980
|
-
case EXECUTION_STATUS.CLEANUP_IN_PROGRESS:
|
|
1981
|
-
return {
|
|
1982
|
-
displayText: "CleanUp in progress",
|
|
1983
|
-
backgroundColor: "#c9e3f9",
|
|
1984
|
-
notificationMessage: "Cleanup is in progress."
|
|
1985
|
-
};
|
|
1986
|
-
case EXECUTION_STATUS.IMPORT_DONE:
|
|
1987
|
-
return {
|
|
1988
|
-
displayText: "Done",
|
|
1989
|
-
backgroundColor: "#d4efd5",
|
|
1990
|
-
notificationMessage: "Import completed successfully."
|
|
1991
|
-
};
|
|
1992
|
-
case EXECUTION_STATUS.CLEANUP_DONE:
|
|
1993
|
-
return {
|
|
1994
|
-
displayText: "Done",
|
|
1995
|
-
backgroundColor: "#d4efd5",
|
|
1996
|
-
notificationMessage: "Cleanup completed successfully."
|
|
1997
|
-
};
|
|
1998
|
-
case EXECUTION_STATUS.ERROR:
|
|
1999
|
-
return {
|
|
2000
|
-
displayText: "Error",
|
|
2001
|
-
backgroundColor: "#fdd2cf",
|
|
2002
|
-
notificationMessage: "Error occurred"
|
|
2003
|
-
};
|
|
2004
|
-
default:
|
|
2005
|
-
throw new Error(`Unknown status constant: ${status}`);
|
|
2006
|
-
}
|
|
2007
|
-
}
|
|
2008
|
-
//----------------------------------------------------------------
|
|
2009
|
-
//---- updateCurrentStatus -----------------------------------------
|
|
2010
|
-
/**
|
|
2011
|
-
* @param {number} status - Status constant
|
|
2012
|
-
*/
|
|
2013
|
-
updateCurrentStatus(status) {
|
|
2014
|
-
const statusProps = this.getStatusProperties(status);
|
|
2015
|
-
this.CurrentStatus.cell.setValue(statusProps.displayText);
|
|
2016
|
-
this.CurrentStatus.cell.setBackground(statusProps.backgroundColor);
|
|
2017
|
-
}
|
|
2018
|
-
//----------------------------------------------------------------
|
|
2019
|
-
//---- updateLastImportDate ----------------------------------------
|
|
2020
|
-
/**
|
|
2021
|
-
* updating the last import attempt date in a config sheet
|
|
2022
|
-
*/
|
|
2023
|
-
updateLastImportDate() {
|
|
2024
|
-
this.LastImportDate.cell.setValue(
|
|
2025
|
-
EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss")
|
|
2026
|
-
);
|
|
2027
|
-
}
|
|
2028
|
-
//----------------------------------------------------------------
|
|
2029
|
-
//---- updateLastRequstedDate --------------------------------------
|
|
2030
|
-
/**
|
|
2031
|
-
* Updating the last requested date in a config sheet
|
|
2032
|
-
* @param date Date requested date
|
|
2033
|
-
*/
|
|
2034
|
-
updateLastRequstedDate(date) {
|
|
2035
|
-
if ("LastRequestedDate" in this && (!this.LastRequestedDate.value || date.getTime() != this.LastRequestedDate.value.getTime())) {
|
|
2036
|
-
this.LastRequestedDate.value = new Date(date.getTime());
|
|
2037
|
-
this.LastRequestedDate.cell.setValue(EnvironmentAdapter.formatDate(date, this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"));
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
//----------------------------------------------------------------
|
|
2041
|
-
//---- updateFieldsSheet -------------------------------------------
|
|
2042
|
-
/**
|
|
2043
|
-
* Updating the content of the fields list to simplify the selection of fields for import
|
|
2044
|
-
* @param connector AbstractSource
|
|
2045
|
-
* @param sheetName string, Fields by default
|
|
2046
|
-
*/
|
|
2047
|
-
updateFieldsSheet(source, sheetName = "Fields") {
|
|
2048
|
-
this.validate();
|
|
2049
|
-
var configSheetStorage = new GoogleSheetsStorage(
|
|
2050
|
-
this.mergeParameters({
|
|
2051
|
-
DestinationSheetName: {
|
|
2052
|
-
// @TODO: make sure it would affect destination sheet for import data. CONFIG is an object. Probably we might dublicate it
|
|
2053
|
-
value: sheetName
|
|
2054
|
-
}
|
|
2055
|
-
}),
|
|
2056
|
-
["id"]
|
|
2057
|
-
);
|
|
2058
|
-
var groups = source.getFieldsSchema();
|
|
2059
|
-
var data = [];
|
|
2060
|
-
for (var groupName in groups) {
|
|
2061
|
-
data.push({
|
|
2062
|
-
"id": groupName,
|
|
2063
|
-
"✔️": groupName,
|
|
2064
|
-
"name": "",
|
|
2065
|
-
"description": `=IF( COUNTIFS(A:A, "${groupName} *", B:B, TRUE) > 0, COUNTIFS(A:A, "${groupName} *", B:B, TRUE), "")`
|
|
2066
|
-
});
|
|
2067
|
-
data.push({ "id": `${groupName} !desc`, "✔️": groups[groupName].description });
|
|
2068
|
-
data.push({ "id": `${groupName} !doc`, "✔️": groups[groupName].documentation });
|
|
2069
|
-
if (groups[groupName].fields) {
|
|
2070
|
-
for (var fieldName in groups[groupName].fields) {
|
|
2071
|
-
data.push({
|
|
2072
|
-
"id": groupName + " " + fieldName,
|
|
2073
|
-
"name": fieldName,
|
|
2074
|
-
"description": groups[groupName].fields[fieldName].description
|
|
2075
|
-
});
|
|
2076
|
-
}
|
|
2077
|
-
data.push({ "id": `${groupName} zzz_separator` });
|
|
2078
|
-
}
|
|
2079
|
-
}
|
|
2080
|
-
for (var groupName in groups) {
|
|
2081
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2082
|
-
if (groupRow !== null) {
|
|
2083
|
-
let depth = configSheetStorage.SHEET.getRowGroupDepth(groupRow.rowIndex + 5);
|
|
2084
|
-
if (depth > 0) {
|
|
2085
|
-
configSheetStorage.SHEET.getRowGroup(groupRow.rowIndex + 5, depth).remove();
|
|
2086
|
-
}
|
|
2087
|
-
}
|
|
2088
|
-
}
|
|
2089
|
-
configSheetStorage.saveData(data);
|
|
2090
|
-
configSheetStorage.SHEET.sort(configSheetStorage.getColumnIndexByName("id"));
|
|
2091
|
-
configSheetStorage.loadDataFromSheet();
|
|
2092
|
-
configSheetStorage.SHEET.hideColumns(configSheetStorage.getColumnIndexByName("id"));
|
|
2093
|
-
configSheetStorage.SHEET.setColumnWidth(configSheetStorage.getColumnIndexByName("✔️"), 25);
|
|
2094
|
-
configSheetStorage.SHEET.autoResizeColumn(3);
|
|
2095
|
-
var checkboxRule = SpreadsheetApp.newDataValidation().requireCheckbox().build();
|
|
2096
|
-
for (var groupName in groups) {
|
|
2097
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2098
|
-
var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 2}`);
|
|
2099
|
-
range.setFontWeight("bold").setBackground("black").setFontColor("white");
|
|
2100
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": `${groupName} !desc` });
|
|
2101
|
-
var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 3}`);
|
|
2102
|
-
range.setBackground("#f3f3f3");
|
|
2103
|
-
var checkboxesColumnIndex = configSheetStorage.getColumnIndexByName("✔️");
|
|
2104
|
-
var groupNameRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2105
|
-
var groupFieldsCount = this.getFieldsCountByGroupName(configSheetStorage, groupName);
|
|
2106
|
-
configSheetStorage.SHEET.getRange(groupNameRow.rowIndex + 5, checkboxesColumnIndex, groupFieldsCount).setDataValidation(checkboxRule);
|
|
2107
|
-
configSheetStorage.SHEET.getRange(`${groupNameRow.rowIndex + 5}:${groupNameRow.rowIndex + 4 + groupFieldsCount}`).shiftRowGroupDepth(1);
|
|
2108
|
-
}
|
|
2109
|
-
configSheetStorage.SHEET.collapseAllRowGroups();
|
|
2110
|
-
}
|
|
2111
|
-
//----------------------------------------------------------------
|
|
2112
|
-
//---- getFieldsCountByGroupName -----------------------------------
|
|
2113
|
-
/**
|
|
2114
|
-
* @param GoogleSheetsConfig object
|
|
2115
|
-
* @param groupName string name of the group
|
|
2116
|
-
* @return integer number of fields of requested group
|
|
2117
|
-
*/
|
|
2118
|
-
getFieldsCountByGroupName(configSheetStorage, groupName) {
|
|
2119
|
-
return Object.values(configSheetStorage.values).filter((element) => element.name && element.id.startsWith(`${groupName} `)).length;
|
|
2120
|
-
}
|
|
2121
|
-
//----------------------------------------------------------------
|
|
2122
|
-
//---- isInProgress ------------------------------------------------
|
|
2123
|
-
/**
|
|
2124
|
-
* Checking current status if it is in progress or not
|
|
2125
|
-
* @return boolean true is process in progress
|
|
2126
|
-
*/
|
|
2127
|
-
isInProgress() {
|
|
2128
|
-
let isInProgress = null;
|
|
2129
|
-
if (this.CurrentStatus.cell.getValue().indexOf("progress") !== -1) {
|
|
2130
|
-
let diff = (/* @__PURE__ */ new Date() - new Date(EnvironmentAdapter.formatDate(this.LastImportDate.cell.getValue(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"))) / (1e3 * 60);
|
|
2131
|
-
isInProgress = diff < this.MaxRunTimeout.value;
|
|
2132
|
-
} else {
|
|
2133
|
-
isInProgress = false;
|
|
2134
|
-
}
|
|
2135
|
-
return isInProgress;
|
|
2136
|
-
}
|
|
2137
|
-
//----------------------------------------------------------------
|
|
2138
|
-
//---- addWarningToCurrentStatus -----------------------------------
|
|
2139
|
-
addWarningToCurrentStatus() {
|
|
2140
|
-
this.CurrentStatus.cell.setBackground("#fff0c4");
|
|
2141
|
-
}
|
|
2142
|
-
//----------------------------------------------------------------
|
|
2143
|
-
//---- validate ----------------------------------------------------
|
|
2144
|
-
/**
|
|
2145
|
-
* validating if google sheets config is correct
|
|
2146
|
-
*/
|
|
2147
|
-
validate() {
|
|
2148
|
-
let scriptTimeZone = Session.getScriptTimeZone();
|
|
2149
|
-
if (scriptTimeZone != "Etc/UTC") {
|
|
2150
|
-
throw new Error(`The Apps Script time zone must be set to UTC to avoid confusion with dates. Currently, it is set to ${scriptTimeZone} instead. Update the time zone in Project Settings`);
|
|
2151
|
-
}
|
|
2152
|
-
super.validate();
|
|
2153
|
-
}
|
|
2154
|
-
//----------------------------------------------------------------
|
|
2155
|
-
//---- logMessage --------------------------------------------------
|
|
2156
|
-
/**
|
|
2157
|
-
* @param string message to Log
|
|
2158
|
-
*/
|
|
2159
|
-
logMessage(message, removeExistingMessage = false) {
|
|
2160
|
-
console.log(message);
|
|
2161
|
-
let formattedDate = EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss");
|
|
2162
|
-
let currentLog = removeExistingMessage ? "" : this.Log.cell.getValue();
|
|
2163
|
-
currentLog ? currentLog += "\n" : "";
|
|
2164
|
-
let emoji = "☑️ ";
|
|
2165
|
-
let match;
|
|
2166
|
-
if (match = message.match(new RegExp("^(\\p{Emoji_Presentation}|\\p{Emoji}\\uFE0F|\\p{Extended_Pictographic})\\s+", "u"))) {
|
|
2167
|
-
emoji = match[0];
|
|
2168
|
-
message = message.slice(2).trim();
|
|
2169
|
-
}
|
|
2170
|
-
this.Log.cell.setValue(
|
|
2171
|
-
`${currentLog}${emoji}${formattedDate}: ${message}`
|
|
2172
|
-
);
|
|
2173
|
-
this.updateLastImportDate();
|
|
2174
|
-
}
|
|
2175
|
-
showCredentialsDialog(source) {
|
|
2176
|
-
const ui = SpreadsheetApp.getUi();
|
|
2177
|
-
const template = HtmlService.createTemplateFromFile("Views/credentials-input-dialog");
|
|
2178
|
-
template.source = source;
|
|
2179
|
-
const html = template.evaluate().setWidth(400).setHeight(450);
|
|
2180
|
-
ui.showModalDialog(html, `${source.constructor.name} Credentials`);
|
|
2181
|
-
}
|
|
2182
|
-
showManualBackfillDialog(source) {
|
|
2183
|
-
const ui = SpreadsheetApp.getUi();
|
|
2184
|
-
const template = HtmlService.createTemplateFromFile("Views/manual-backfill-dialog");
|
|
2185
|
-
template.source = source;
|
|
2186
|
-
const html = template.evaluate().setWidth(600).setHeight(300);
|
|
2187
|
-
ui.showModalDialog(html, "Manual Backfill");
|
|
2188
|
-
}
|
|
2189
|
-
//---- sendNotifications -------------------------------------------
|
|
2190
|
-
/**
|
|
2191
|
-
* Send notifications based on configuration settings
|
|
2192
|
-
* @param {Object} params - Parameters object
|
|
2193
|
-
* @param {string} params.status - Current status value
|
|
2194
|
-
* @param {string} params.error - Error message for Error status
|
|
2195
|
-
*/
|
|
2196
|
-
sendNotifications({ status, error }) {
|
|
2197
|
-
try {
|
|
2198
|
-
const { title, messageWithDetails } = this.prepareNotificationContent({ status, error });
|
|
2199
|
-
if (this.NotifyByEmail && this.NotifyByEmail.value && this.NotifyByEmail.value.trim()) {
|
|
2200
|
-
EmailNotification.send({
|
|
2201
|
-
to: this.NotifyByEmail.value,
|
|
2202
|
-
subject: title,
|
|
2203
|
-
message: messageWithDetails
|
|
2204
|
-
});
|
|
2205
|
-
}
|
|
2206
|
-
if (this.NotifyByGoogleChat && this.NotifyByGoogleChat.value && this.NotifyByGoogleChat.value.trim()) {
|
|
2207
|
-
GoogleChatNotification.send({
|
|
2208
|
-
webhookUrl: this.NotifyByGoogleChat.value.trim(),
|
|
2209
|
-
message: messageWithDetails
|
|
2210
|
-
});
|
|
2211
|
-
}
|
|
2212
|
-
} catch (error2) {
|
|
2213
|
-
this.logMessage(`⚠️ Notification error: ${error2.message}`);
|
|
2214
|
-
}
|
|
2215
|
-
}
|
|
2216
|
-
//----------------------------------------------------------------
|
|
2217
|
-
//---- prepareNotificationContent ----------------------------------
|
|
2218
|
-
/**
|
|
2219
|
-
* Prepare notification title and message content
|
|
2220
|
-
* @param {Object} params - Parameters object
|
|
2221
|
-
* @param {number} params.status - Status constant
|
|
2222
|
-
* @param {string} params.error - Error message for Error status
|
|
2223
|
-
* @returns {Object} - Object with title and messageWithDetails properties
|
|
2224
|
-
*/
|
|
2225
|
-
prepareNotificationContent({ status, error }) {
|
|
2226
|
-
const documentName = this.configSpreadsheet.getName();
|
|
2227
|
-
const documentUrl = this.configSpreadsheet.getUrl();
|
|
2228
|
-
const { displayText, notificationMessage } = this.getStatusProperties(status);
|
|
2229
|
-
const title = `${documentName} - Status: ${displayText}`;
|
|
2230
|
-
return {
|
|
2231
|
-
title,
|
|
2232
|
-
messageWithDetails: `${title}
|
|
2233
|
-
${documentUrl}
|
|
2234
|
-
|
|
2235
|
-
${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}` : ""}`
|
|
2236
|
-
};
|
|
2237
|
-
}
|
|
2238
|
-
//----------------------------------------------------------------
|
|
2239
|
-
//---- shouldSendNotifications -------------------------------------
|
|
2240
|
-
/**
|
|
2241
|
-
* Determine if notifications should be sent based on status and filter setting
|
|
2242
|
-
* @param {number} status - Status constant
|
|
2243
|
-
* @returns {boolean} - True if notifications should be sent
|
|
2244
|
-
*/
|
|
2245
|
-
shouldSendNotifications(status) {
|
|
2246
|
-
var _a;
|
|
2247
|
-
const notifyWhen = (_a = this.NotifyWhen) == null ? void 0 : _a.value;
|
|
2248
|
-
switch (notifyWhen) {
|
|
2249
|
-
case "On error":
|
|
2250
|
-
return status === EXECUTION_STATUS.ERROR;
|
|
2251
|
-
case "On success":
|
|
2252
|
-
return status === EXECUTION_STATUS.IMPORT_DONE;
|
|
2253
|
-
case "Always":
|
|
2254
|
-
return status === EXECUTION_STATUS.ERROR || status === EXECUTION_STATUS.IMPORT_DONE;
|
|
2255
|
-
case "Never":
|
|
2256
|
-
case "":
|
|
2257
|
-
default:
|
|
2258
|
-
return false;
|
|
2259
|
-
}
|
|
2260
|
-
}
|
|
2261
|
-
//----------------------------------------------------------------
|
|
2262
|
-
//---- createTimeoutTrigger ----------------------------------------
|
|
2263
|
-
createTimeoutTrigger() {
|
|
2264
|
-
this.removeTimeoutTrigger();
|
|
2265
|
-
ScriptApp.newTrigger("checkForTimeout").timeBased().after((this.MaxRunTimeout.value * 2 + 1) * 60 * 1e3).create();
|
|
2266
|
-
}
|
|
2267
|
-
//----------------------------------------------------------------
|
|
2268
|
-
//---- removeTimeoutTrigger ----------------------------------------
|
|
2269
|
-
removeTimeoutTrigger() {
|
|
2270
|
-
const triggers = ScriptApp.getProjectTriggers();
|
|
2271
|
-
let removedCount = 0;
|
|
2272
|
-
triggers.forEach((trigger) => {
|
|
2273
|
-
if (trigger.getHandlerFunction() === "checkForTimeout") {
|
|
2274
|
-
ScriptApp.deleteTrigger(trigger);
|
|
2275
|
-
removedCount++;
|
|
2276
|
-
}
|
|
2277
|
-
});
|
|
2278
|
-
console.log(`[TimeoutTrigger] ${removedCount > 0 ? `Removed ${removedCount} timeout trigger(s)` : "No timeout triggers found to remove"}`);
|
|
2279
|
-
}
|
|
2280
|
-
//----------------------------------------------------------------
|
|
2281
|
-
//---- checkForTimeout ---------------------------------------------
|
|
2282
|
-
checkForTimeout() {
|
|
2283
|
-
if (!this.isInProgress()) {
|
|
2284
|
-
console.log("[TimeoutTrigger] Status is NOT in progress, setting to Error and sending notification");
|
|
2285
|
-
this.handleStatusUpdate({
|
|
2286
|
-
status: EXECUTION_STATUS.ERROR,
|
|
2287
|
-
error: "Import was interrupted (likely due to timeout)"
|
|
2288
|
-
});
|
|
2289
|
-
} else {
|
|
2290
|
-
console.log("[TimeoutTrigger] Status is still in progress");
|
|
2291
|
-
}
|
|
2292
|
-
}
|
|
2293
|
-
//----------------------------------------------------------------
|
|
2294
|
-
};
|
|
2295
|
-
var GoogleChatNotification = class GoogleChatNotification2 {
|
|
2296
|
-
/**
|
|
2297
|
-
* Send Google Chat notification
|
|
2298
|
-
* @param {Object} params - Parameters object
|
|
2299
|
-
* @param {string} params.webhookUrl - Google Chat webhook URL
|
|
2300
|
-
* @param {string} params.message - Formatted notification message
|
|
2301
|
-
*/
|
|
2302
|
-
static send(params) {
|
|
2303
|
-
const { webhookUrl, message } = params;
|
|
2304
|
-
if (!webhookUrl || !webhookUrl.trim()) {
|
|
2305
|
-
return;
|
|
2306
|
-
}
|
|
2307
|
-
try {
|
|
2308
|
-
const response = EnvironmentAdapter.fetch(webhookUrl.trim(), {
|
|
2309
|
-
method: "POST",
|
|
2310
|
-
headers: {
|
|
2311
|
-
"Content-Type": "application/json; charset=UTF-8"
|
|
2312
|
-
},
|
|
2313
|
-
payload: JSON.stringify({ text: message })
|
|
2314
|
-
});
|
|
2315
|
-
if (response.getResponseCode() === 200) {
|
|
2316
|
-
console.log("Google Chat notification sent successfully");
|
|
2317
|
-
} else {
|
|
2318
|
-
console.error(`Google Chat notification failed with status: ${response.getResponseCode()}`);
|
|
2319
|
-
}
|
|
2320
|
-
} catch (error) {
|
|
2321
|
-
console.error("Failed to send Google Chat notification:", error);
|
|
2322
|
-
}
|
|
2323
|
-
}
|
|
2324
|
-
};
|
|
2325
|
-
var EmailNotification = class EmailNotification2 {
|
|
2326
|
-
/**
|
|
2327
|
-
* Send email notification
|
|
2328
|
-
* @param {Object} params - Parameters object
|
|
2329
|
-
* @param {string} params.to - Email address(es) to send to (can be multiple separated by commas)
|
|
2330
|
-
* @param {string} params.subject - Email subject line
|
|
2331
|
-
* @param {string} params.message - Formatted notification message
|
|
2332
|
-
*/
|
|
2333
|
-
static send(params) {
|
|
2334
|
-
const { to, subject, message } = params;
|
|
2335
|
-
if (!to || !to.trim()) {
|
|
2336
|
-
return;
|
|
2337
|
-
}
|
|
2338
|
-
try {
|
|
2339
|
-
const emailAddresses = to.split(",").map((email) => email.trim()).filter((email) => email.length > 0).filter((email) => this.isValidEmail(email)).join(",");
|
|
2340
|
-
if (!emailAddresses) {
|
|
2341
|
-
console.log("Email notification skipped: no valid email addresses found");
|
|
2342
|
-
return;
|
|
2343
|
-
}
|
|
2344
|
-
MailApp.sendEmail(
|
|
2345
|
-
emailAddresses,
|
|
2346
|
-
subject,
|
|
2347
|
-
message,
|
|
2348
|
-
{ noReply: true }
|
|
2349
|
-
);
|
|
2350
|
-
console.log(`Email notification sent successfully to: ${emailAddresses}`);
|
|
2351
|
-
} catch (error) {
|
|
2352
|
-
console.error(`Failed to send email notification: ${error.message}`);
|
|
2353
|
-
}
|
|
2354
|
-
}
|
|
2355
|
-
/**
|
|
2356
|
-
* Validate email address format
|
|
2357
|
-
* @param {string} email - Email address to validate
|
|
2358
|
-
* @returns {boolean} - True if email is valid
|
|
2359
|
-
*/
|
|
2360
|
-
static isValidEmail(email) {
|
|
2361
|
-
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
|
2362
|
-
const isValid = emailRegex.test(email);
|
|
2363
|
-
if (!isValid) {
|
|
2364
|
-
console.log(`Invalid email address skipped: ${email}`);
|
|
2365
|
-
}
|
|
2366
|
-
return isValid;
|
|
2367
|
-
}
|
|
2368
|
-
};
|
|
2369
1773
|
const Core = {
|
|
2370
1774
|
AbstractException,
|
|
2371
1775
|
HttpRequestException,
|
|
2372
1776
|
UnsupportedEnvironmentException,
|
|
2373
|
-
EnvironmentAdapter,
|
|
2374
1777
|
AbstractStorage,
|
|
2375
1778
|
AbstractSource,
|
|
2376
1779
|
AbstractRunConfig,
|
|
2377
1780
|
AbstractConnector,
|
|
2378
1781
|
AbstractConfig,
|
|
1782
|
+
HttpUtils,
|
|
1783
|
+
FileUtils,
|
|
1784
|
+
DateUtils,
|
|
1785
|
+
CryptoUtils,
|
|
1786
|
+
AsyncUtils,
|
|
2379
1787
|
RunConfigDto,
|
|
2380
1788
|
SourceConfigDto,
|
|
2381
1789
|
StorageConfigDto,
|
|
2382
1790
|
ConfigDto,
|
|
2383
1791
|
NodeJsConfig,
|
|
2384
|
-
GoogleSheetsConfig,
|
|
2385
|
-
GoogleChatNotification,
|
|
2386
|
-
EmailNotification,
|
|
2387
1792
|
HTTP_STATUS,
|
|
2388
|
-
ENVIRONMENT,
|
|
2389
1793
|
EXECUTION_STATUS,
|
|
2390
1794
|
RUN_CONFIG_TYPE,
|
|
2391
1795
|
CONFIG_ATTRIBUTES
|
|
2392
1796
|
};
|
|
2393
|
-
const GoogleSheets = (function() {
|
|
2394
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter3, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
2395
|
-
var GoogleSheetsStorage2 = class GoogleSheetsStorage extends AbstractStorage2 {
|
|
2396
|
-
//---- constructor -------------------------------------------------
|
|
2397
|
-
/**
|
|
2398
|
-
* Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
|
|
2399
|
-
* @param config (object) instance of AbscractConfig
|
|
2400
|
-
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2401
|
-
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2402
|
-
*/
|
|
2403
|
-
constructor(config, uniqueKeyColumns, schema = null) {
|
|
2404
|
-
super(
|
|
2405
|
-
config.mergeParameters({
|
|
2406
|
-
CleanUpToKeepWindow: {
|
|
2407
|
-
requiredType: "number"
|
|
2408
|
-
},
|
|
2409
|
-
DestinationSheetName: {
|
|
2410
|
-
isRequired: true,
|
|
2411
|
-
default: "Data"
|
|
2412
|
-
}
|
|
2413
|
-
}),
|
|
2414
|
-
uniqueKeyColumns,
|
|
2415
|
-
schema
|
|
2416
|
-
);
|
|
2417
|
-
this.SHEET = this.getDestinationSheet(config);
|
|
2418
|
-
this.loadDataFromSheet();
|
|
2419
|
-
}
|
|
2420
|
-
//----------------------------------------------------------------
|
|
2421
|
-
//---- loadDataFromSheet -------------------------------------------
|
|
2422
|
-
/**
|
|
2423
|
-
* reading Data from the source sheet and loading it to this.values
|
|
2424
|
-
*/
|
|
2425
|
-
loadDataFromSheet() {
|
|
2426
|
-
const values = this.SHEET.getDataRange().getValues();
|
|
2427
|
-
this.columnNames = values.shift();
|
|
2428
|
-
if (this.uniqueKeyColumns.some((column) => !this.columnNames.includes(column))) {
|
|
2429
|
-
throw new Error(`Sheet '${this.SHEET.getName()}' is missing one the folling columns required for unique key: column '${this.uniqueKeyColumns}'`);
|
|
2430
|
-
}
|
|
2431
|
-
this.values = values.reduce((acc, row, rowIndex) => {
|
|
2432
|
-
const uniqueKey = this.uniqueKeyColumns.reduce((accumulator, columnName) => {
|
|
2433
|
-
let index = this.columnNames.indexOf(columnName);
|
|
2434
|
-
accumulator += `|${row[index]}`;
|
|
2435
|
-
return accumulator;
|
|
2436
|
-
}, []);
|
|
2437
|
-
acc[uniqueKey] = {
|
|
2438
|
-
...this.columnNames.reduce((obj, col, colIndex) => ({
|
|
2439
|
-
...obj,
|
|
2440
|
-
[col]: row[colIndex]
|
|
2441
|
-
}), {}),
|
|
2442
|
-
rowIndex
|
|
2443
|
-
// Add row index for reference
|
|
2444
|
-
};
|
|
2445
|
-
return acc;
|
|
2446
|
-
}, {});
|
|
2447
|
-
this.addedRecordsBuffer = [];
|
|
2448
|
-
}
|
|
2449
|
-
//----------------------------------------------------------------
|
|
2450
|
-
//---- getDestinationSheet -----------------------------------------
|
|
2451
|
-
/**
|
|
2452
|
-
* @param object destination Spreadsheet Config
|
|
2453
|
-
* @param object destination Sheet Name Config
|
|
2454
|
-
* @return Sheet object for data Destination
|
|
2455
|
-
*/
|
|
2456
|
-
getDestinationSheet(config) {
|
|
2457
|
-
if (!this.SHEET) {
|
|
2458
|
-
if (!config.DestinationSpreadsheet || !config.DestinationSpreadsheet.value) {
|
|
2459
|
-
config.DestinationSpreadsheet = { "spreadsheet": config.configSpreadsheet };
|
|
2460
|
-
} else {
|
|
2461
|
-
let match = config.DestinationSpreadsheet.value.match(/\/d\/([a-zA-Z0-9-_]+)/);
|
|
2462
|
-
if (match && match[1]) {
|
|
2463
|
-
config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match[1]);
|
|
2464
|
-
} else {
|
|
2465
|
-
let match2 = config.DestinationSpreadsheet.cell.getRichTextValue().getLinkUrl().match(/\/d\/([a-zA-Z0-9-_]+)/);
|
|
2466
|
-
if (match2 && match2[1]) {
|
|
2467
|
-
config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match2[1]);
|
|
2468
|
-
} else {
|
|
2469
|
-
throw new Error(`Destination Spreadsheet must be specified in config either by Spreadsheet Id or by a link to spreadsheet`);
|
|
2470
|
-
}
|
|
2471
|
-
}
|
|
2472
|
-
}
|
|
2473
|
-
if (config.DestinationSpreadsheet.spreadsheet == null) {
|
|
2474
|
-
this.config.logMessage(`Cannot load destination sheet document`);
|
|
2475
|
-
}
|
|
2476
|
-
config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.getSheetByName(
|
|
2477
|
-
config.DestinationSheetName.value
|
|
2478
|
-
);
|
|
2479
|
-
if (!config.DestinationSpreadsheet.sheet) {
|
|
2480
|
-
config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.insertSheet(
|
|
2481
|
-
config.DestinationSheetName.value,
|
|
2482
|
-
config.DestinationSpreadsheet.spreadsheet.getSheets().length
|
|
2483
|
-
);
|
|
2484
|
-
this.config.logMessage(`Sheet '${config.DestinationSheetName.value}' was created.`);
|
|
2485
|
-
}
|
|
2486
|
-
this.SHEET = config.DestinationSpreadsheet.sheet;
|
|
2487
|
-
if (this.isEmpty()) {
|
|
2488
|
-
this.addHeader(this.uniqueKeyColumns);
|
|
2489
|
-
this.config.logMessage(`Columns for unique keys were added to '${config.DestinationSheetName.value}' sheet.`);
|
|
2490
|
-
}
|
|
2491
|
-
}
|
|
2492
|
-
return this.SHEET;
|
|
2493
|
-
}
|
|
2494
|
-
//----------------------------------------------------------------
|
|
2495
|
-
//---- addRecord ---------------------------------------------------
|
|
2496
|
-
/**
|
|
2497
|
-
* Checking if record exists by id
|
|
2498
|
-
* @param object {record} object with record data
|
|
2499
|
-
* @param Boolean {useBuffer}: Set to `false` if the record must be saved instantly, or `true` to save it later using `this.saveAddedRecordsFromBuffer()`.
|
|
2500
|
-
* @return object {record} with added rowIndex property
|
|
2501
|
-
*/
|
|
2502
|
-
addRecord(record, useBuffer = false) {
|
|
2503
|
-
record = this.stringifyNeastedFields(record);
|
|
2504
|
-
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2505
|
-
if (useBuffer) {
|
|
2506
|
-
this.addedRecordsBuffer[uniqueKey] = record;
|
|
2507
|
-
} else {
|
|
2508
|
-
let data = this.columnNames.map((key) => record[key] || "");
|
|
2509
|
-
this.SHEET.appendRow(data);
|
|
2510
|
-
record.rowIndex = this.SHEET.getLastRow() - 2;
|
|
2511
|
-
this.values[uniqueKey] = record;
|
|
2512
|
-
}
|
|
2513
|
-
return record;
|
|
2514
|
-
}
|
|
2515
|
-
//----------------------------------------------------------------
|
|
2516
|
-
//---- saveData ----------------------------------------------------
|
|
2517
|
-
/**
|
|
2518
|
-
* Saving data to a storage
|
|
2519
|
-
* @param {data} array of assoc objects with records to save
|
|
2520
|
-
*/
|
|
2521
|
-
saveData(data) {
|
|
2522
|
-
var recordsAdded = 0;
|
|
2523
|
-
var recordsUpdated = 0;
|
|
2524
|
-
data.map((row) => {
|
|
2525
|
-
let newFields = Object.keys(row).filter((column2) => !this.columnNames.includes(column2));
|
|
2526
|
-
for (var column in newFields) {
|
|
2527
|
-
this.addColumn(newFields[column], this.columnNames.length + 1);
|
|
2528
|
-
this.config.logMessage(`Column '${newFields[column]}' was added to '${this.SHEET.getName()}' sheet`);
|
|
2529
|
-
}
|
|
2530
|
-
if (this.isRecordExists(row)) {
|
|
2531
|
-
if (this.updateRecord(row)) {
|
|
2532
|
-
recordsUpdated++;
|
|
2533
|
-
}
|
|
2534
|
-
} else {
|
|
2535
|
-
this.addRecord(row, true);
|
|
2536
|
-
recordsAdded += this.saveRecordsAddedToBuffer(100);
|
|
2537
|
-
}
|
|
2538
|
-
});
|
|
2539
|
-
recordsAdded += this.saveRecordsAddedToBuffer(0);
|
|
2540
|
-
if (recordsAdded > 0) {
|
|
2541
|
-
this.config.logMessage(`${recordsAdded} records were added`);
|
|
2542
|
-
}
|
|
2543
|
-
if (recordsUpdated > 0) {
|
|
2544
|
-
this.config.logMessage(`${recordsUpdated} records were updated`);
|
|
2545
|
-
}
|
|
2546
|
-
}
|
|
2547
|
-
//----------------------------------------------------------------
|
|
2548
|
-
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
2549
|
-
/**
|
|
2550
|
-
* Add records from buffer to a sheet
|
|
2551
|
-
* @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
|
|
2552
|
-
*/
|
|
2553
|
-
saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2554
|
-
let recordsAdded = 0;
|
|
2555
|
-
let bufferSize = Object.keys(this.addedRecordsBuffer).length;
|
|
2556
|
-
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
2557
|
-
let startIndex = this.SHEET.getLastRow() - 2;
|
|
2558
|
-
let index = 1;
|
|
2559
|
-
let data = [];
|
|
2560
|
-
for (var uniqueKey in this.addedRecordsBuffer) {
|
|
2561
|
-
let record = this.addedRecordsBuffer[uniqueKey];
|
|
2562
|
-
record.rowIndex = startIndex + index++;
|
|
2563
|
-
this.values[uniqueKey] = record;
|
|
2564
|
-
data.push(this.columnNames.map((key) => record[key] || ""));
|
|
2565
|
-
}
|
|
2566
|
-
this.SHEET.getRange(startIndex + 3, 1, data.length, data[0].length).setValues(data);
|
|
2567
|
-
recordsAdded = bufferSize;
|
|
2568
|
-
this.addedRecordsBuffer = {};
|
|
2569
|
-
}
|
|
2570
|
-
return recordsAdded;
|
|
2571
|
-
}
|
|
2572
|
-
//----------------------------------------------------------------
|
|
2573
|
-
//---- updateRecord ------------------------------------------------
|
|
2574
|
-
/**
|
|
2575
|
-
* Update content of an existing record
|
|
2576
|
-
* @param object {record} object with record data
|
|
2577
|
-
* @return boolean Returns true if the record was updated; otherwise, returns false
|
|
2578
|
-
*/
|
|
2579
|
-
updateRecord(record) {
|
|
2580
|
-
record = this.stringifyNeastedFields(record);
|
|
2581
|
-
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2582
|
-
var existingRecord = this.getRecordByUniqueKey(uniqueKey);
|
|
2583
|
-
var isRecordUpdated = false;
|
|
2584
|
-
this.columnNames.forEach((columnName, columnIndex) => {
|
|
2585
|
-
if (columnName in record && !this.areValuesEqual(record[columnName], existingRecord[columnName])) {
|
|
2586
|
-
console.log(`${uniqueKey}: ${existingRecord[columnName]} ${typeof existingRecord[columnName]} → ${record[columnName]} ${typeof record[columnName]}`);
|
|
2587
|
-
this.SHEET.getRange(existingRecord.rowIndex + 2, columnIndex + 1, 1, 1).setValue(record[columnName]);
|
|
2588
|
-
existingRecord[columnName] = record[columnName];
|
|
2589
|
-
isRecordUpdated = true;
|
|
2590
|
-
}
|
|
2591
|
-
});
|
|
2592
|
-
return isRecordUpdated;
|
|
2593
|
-
}
|
|
2594
|
-
//----------------------------------------------------------------
|
|
2595
|
-
//---- deleteRecord ------------------------------------------------
|
|
2596
|
-
/**
|
|
2597
|
-
* Delete record from a sheet
|
|
2598
|
-
* @param uniqueKey {string} unique key of the record to delete
|
|
2599
|
-
*/
|
|
2600
|
-
deleteRecord(uniqueKey) {
|
|
2601
|
-
if (!(uniqueKey in this.values)) {
|
|
2602
|
-
throw new Error(`Unable to delete the record with ID ${uniqueKey} because it was not found`);
|
|
2603
|
-
} else if (!("rowIndex" in this.values[uniqueKey])) {
|
|
2604
|
-
throw new Error(`Unable to delete the record with ID ${uniqueKey} because it does not have a rowIndex`);
|
|
2605
|
-
} else {
|
|
2606
|
-
let rowIndex = this.values[uniqueKey].rowIndex;
|
|
2607
|
-
this.SHEET.deleteRow(rowIndex + 2);
|
|
2608
|
-
for (uniqueKey in this.values) {
|
|
2609
|
-
if (this.values[uniqueKey].rowIndex > rowIndex) {
|
|
2610
|
-
this.values[uniqueKey].rowIndex--;
|
|
2611
|
-
}
|
|
2612
|
-
}
|
|
2613
|
-
}
|
|
2614
|
-
}
|
|
2615
|
-
//----------------------------------------------------------------
|
|
2616
|
-
//---- addHeader ---------------------------------------------------
|
|
2617
|
-
/**
|
|
2618
|
-
* Adding header to sheet
|
|
2619
|
-
* @param target Sheet
|
|
2620
|
-
* @param array column names to be added
|
|
2621
|
-
*/
|
|
2622
|
-
addHeader(columnNames) {
|
|
2623
|
-
columnNames.forEach((columnName, index) => {
|
|
2624
|
-
this.addColumn(columnName, index + 1);
|
|
2625
|
-
});
|
|
2626
|
-
this.SHEET.getRange("1:1").setBackground("#f3f3f3").setHorizontalAlignment("center");
|
|
2627
|
-
this.SHEET.setFrozenRows(1);
|
|
2628
|
-
}
|
|
2629
|
-
//----------------------------------------------------------------
|
|
2630
|
-
//---- addColumn ---------------------------------------------------
|
|
2631
|
-
/**
|
|
2632
|
-
* Adding a column to the sheet
|
|
2633
|
-
* @param columnName (string) column name
|
|
2634
|
-
* @param columnIndex (integer) optional; column index
|
|
2635
|
-
*/
|
|
2636
|
-
addColumn(columnName, columnIndex = 1) {
|
|
2637
|
-
const numColumns = this.SHEET.getMaxColumns();
|
|
2638
|
-
if (columnIndex <= 0 || columnIndex > numColumns + 1) {
|
|
2639
|
-
throw new Error(`Column index ${columnIndex} is out of bounds (1-${numColumns + 1})`);
|
|
2640
|
-
}
|
|
2641
|
-
if (columnIndex <= numColumns) {
|
|
2642
|
-
const headerValue = this.SHEET.getRange(1, columnIndex).getValue();
|
|
2643
|
-
if (headerValue !== "") {
|
|
2644
|
-
const findFirstEmptyColumn = (startIndex) => {
|
|
2645
|
-
let index = startIndex;
|
|
2646
|
-
let foundEmpty = false;
|
|
2647
|
-
while (index <= numColumns) {
|
|
2648
|
-
if (this.SHEET.getRange(1, index).getValue() === "") {
|
|
2649
|
-
foundEmpty = true;
|
|
2650
|
-
break;
|
|
2651
|
-
}
|
|
2652
|
-
index++;
|
|
2653
|
-
}
|
|
2654
|
-
return {
|
|
2655
|
-
columnIndex: index,
|
|
2656
|
-
foundEmpty
|
|
2657
|
-
};
|
|
2658
|
-
};
|
|
2659
|
-
const result = findFirstEmptyColumn(columnIndex);
|
|
2660
|
-
if (!result.foundEmpty) {
|
|
2661
|
-
this.SHEET.insertColumnAfter(numColumns);
|
|
2662
|
-
columnIndex = numColumns + 1;
|
|
2663
|
-
} else {
|
|
2664
|
-
this.SHEET.insertColumnBefore(result.columnIndex);
|
|
2665
|
-
columnIndex = result.columnIndex;
|
|
2666
|
-
}
|
|
2667
|
-
}
|
|
2668
|
-
} else {
|
|
2669
|
-
this.SHEET.insertColumnAfter(numColumns);
|
|
2670
|
-
columnIndex = numColumns + 1;
|
|
2671
|
-
}
|
|
2672
|
-
this.SHEET.getRange(1, columnIndex).setValue(columnName);
|
|
2673
|
-
if (this.schema != null && columnName in this.schema && "GoogleSheetsFormat" in this.schema[columnName]) {
|
|
2674
|
-
let columnLetter = String.fromCharCode(64 + columnIndex);
|
|
2675
|
-
console.log(
|
|
2676
|
-
columnName,
|
|
2677
|
-
this.schema[columnName]["GoogleSheetsFormat"],
|
|
2678
|
-
this.SHEET.getRange(`${columnLetter}:${columnLetter}`).getA1Notation()
|
|
2679
|
-
);
|
|
2680
|
-
this.SHEET.getRange(`${columnLetter}:${columnLetter}`).setNumberFormat(this.schema[columnName]["GoogleSheetsFormat"]);
|
|
2681
|
-
}
|
|
2682
|
-
this.columnNames.push(columnName);
|
|
2683
|
-
}
|
|
2684
|
-
//----------------------------------------------------------------
|
|
2685
|
-
//---- formatColumn ------------------------------------------------
|
|
2686
|
-
/**
|
|
2687
|
-
* Format column as it described in schema
|
|
2688
|
-
* @param columnName (string) column name
|
|
2689
|
-
*/
|
|
2690
|
-
formatColumn(columnName) {
|
|
2691
|
-
if ("type" in this.schema[columnName]) ;
|
|
2692
|
-
}
|
|
2693
|
-
//----------------------------------------------------------------
|
|
2694
|
-
//---- getColumnIndexByName ----------------------------------------
|
|
2695
|
-
/**
|
|
2696
|
-
* @param columnName string column name
|
|
2697
|
-
* @return integer columnIndex
|
|
2698
|
-
*/
|
|
2699
|
-
getColumnIndexByName(columnName) {
|
|
2700
|
-
const columnIndex = this.columnNames.indexOf(columnName);
|
|
2701
|
-
if (columnIndex == -1) {
|
|
2702
|
-
throw new Error(`Column ${columnName} not found in '${this.SHEET.getName()}' sheet`);
|
|
2703
|
-
}
|
|
2704
|
-
return columnIndex + 1;
|
|
2705
|
-
}
|
|
2706
|
-
//----------------------------------------------------------------
|
|
2707
|
-
//---- isEmpty -----------------------------------------------------
|
|
2708
|
-
/**
|
|
2709
|
-
* @return boolean true if sheet is empty, false overwise
|
|
2710
|
-
*/
|
|
2711
|
-
isEmpty() {
|
|
2712
|
-
return this.SHEET.getLastRow() === 0 && this.SHEET.getLastColumn() === 0;
|
|
2713
|
-
}
|
|
2714
|
-
//----------------------------------------------------------------
|
|
2715
|
-
//---- areValuesEqual ----------------------------------------------
|
|
2716
|
-
/**
|
|
2717
|
-
* Comparing to vaariables if they are equal or not
|
|
2718
|
-
* @param value1 (mixed)
|
|
2719
|
-
* @param value2 (mixed)
|
|
2720
|
-
* @return boolean true if equal, falst overwise
|
|
2721
|
-
*/
|
|
2722
|
-
areValuesEqual(value1, value2) {
|
|
2723
|
-
var equal = null;
|
|
2724
|
-
if (typeof value1 !== "undefined" && typeof value2 !== "undefined" && ((value1 == null ? void 0 : value1.constructor.name) == "Date" || (value2 == null ? void 0 : value2.constructor.name) == "Date")) {
|
|
2725
|
-
const normalizeToDate = (value) => {
|
|
2726
|
-
if (value === null || value === "") return null;
|
|
2727
|
-
if (value.constructor.name == "Date") return value;
|
|
2728
|
-
const date = new Date(value);
|
|
2729
|
-
return isNaN(date.getTime()) ? null : date;
|
|
2730
|
-
};
|
|
2731
|
-
const date1 = normalizeToDate(value1);
|
|
2732
|
-
const date2 = normalizeToDate(value2);
|
|
2733
|
-
if (date1 === null || date2 === null) {
|
|
2734
|
-
return value1 === value2;
|
|
2735
|
-
}
|
|
2736
|
-
equal = date1.getTime() === date2.getTime();
|
|
2737
|
-
} else if (typeof value1 == typeof value2) {
|
|
2738
|
-
equal = value1 === value2;
|
|
2739
|
-
} else if (value1 === void 0 && value2 === "" || value2 === void 0 && value1 === "") {
|
|
2740
|
-
equal = true;
|
|
2741
|
-
} else if (value1 === null && value2 === "" || value2 === null && value1 === "") {
|
|
2742
|
-
equal = true;
|
|
2743
|
-
}
|
|
2744
|
-
return equal;
|
|
2745
|
-
}
|
|
2746
|
-
//----------------------------------------------------------------
|
|
2747
|
-
//---- areHeadersNeeded ------------------------------------------
|
|
2748
|
-
/**
|
|
2749
|
-
* Checks if storage is empty and adds headers if needed
|
|
2750
|
-
* if destination sheet is empty than header should be created based on unique key columns list
|
|
2751
|
-
* @return {boolean} true if headers were added, false if they already existed
|
|
2752
|
-
*/
|
|
2753
|
-
areHeadersNeeded() {
|
|
2754
|
-
return this.isEmpty();
|
|
2755
|
-
}
|
|
2756
|
-
//----------------------------------------------------------------
|
|
2757
|
-
};
|
|
2758
|
-
const manifest = {
|
|
2759
|
-
"name": "GoogleSheetsStorage",
|
|
2760
|
-
"description": "Storage for Google Sheets",
|
|
2761
|
-
"title": "Google Sheets",
|
|
2762
|
-
"version": "0.0.0",
|
|
2763
|
-
"author": "OWOX, Inc.",
|
|
2764
|
-
"license": "MIT",
|
|
2765
|
-
"environment": {
|
|
2766
|
-
"node": {
|
|
2767
|
-
"enabled": false
|
|
2768
|
-
},
|
|
2769
|
-
"appscript": {
|
|
2770
|
-
"enabled": true
|
|
2771
|
-
}
|
|
2772
|
-
}
|
|
2773
|
-
};
|
|
2774
|
-
return {
|
|
2775
|
-
GoogleSheetsStorage: GoogleSheetsStorage2,
|
|
2776
|
-
manifest
|
|
2777
|
-
};
|
|
2778
|
-
})();
|
|
2779
1797
|
const GoogleBigQuery = (function() {
|
|
2780
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
1798
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
2781
1799
|
var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
|
|
2782
1800
|
//---- constructor -------------------------------------------------
|
|
2783
1801
|
/**
|
|
2784
|
-
*
|
|
2785
|
-
*
|
|
2786
|
-
* @param config (object) instance of
|
|
1802
|
+
* Abstract class for Google BigQuery storage operations
|
|
1803
|
+
*
|
|
1804
|
+
* @param config (object) instance of AbstractConfig
|
|
2787
1805
|
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2788
1806
|
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2789
1807
|
* @param description (string) string with storage description }
|
|
@@ -2829,22 +1847,29 @@ const GoogleBigQuery = (function() {
|
|
|
2829
1847
|
schema,
|
|
2830
1848
|
description
|
|
2831
1849
|
);
|
|
2832
|
-
this.checkIfGoogleBigQueryIsConnected();
|
|
2833
|
-
this.loadTableSchema();
|
|
2834
1850
|
this.updatedRecordsBuffer = {};
|
|
2835
1851
|
this.totalRecordsProcessed = 0;
|
|
2836
1852
|
}
|
|
1853
|
+
//---- init --------------------------------------------------------
|
|
1854
|
+
/**
|
|
1855
|
+
* Initializing storage
|
|
1856
|
+
*/
|
|
1857
|
+
async init() {
|
|
1858
|
+
this.checkIfGoogleBigQueryIsConnected();
|
|
1859
|
+
await this.loadTableSchema();
|
|
1860
|
+
}
|
|
1861
|
+
//----------------------------------------------------------------
|
|
2837
1862
|
//---- loads Google BigQuery Table Schema ---------------------------
|
|
2838
|
-
loadTableSchema() {
|
|
1863
|
+
async loadTableSchema() {
|
|
2839
1864
|
this.existingColumns = this.getAListOfExistingColumns() || {};
|
|
2840
1865
|
if (Object.keys(this.existingColumns).length == 0) {
|
|
2841
|
-
this.createDatasetIfItDoesntExist();
|
|
2842
|
-
this.existingColumns = this.createTableIfItDoesntExist();
|
|
1866
|
+
await this.createDatasetIfItDoesntExist();
|
|
1867
|
+
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
2843
1868
|
} else {
|
|
2844
1869
|
let selectedFields = this.getSelectedFields();
|
|
2845
1870
|
let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2846
1871
|
if (newFields.length > 0) {
|
|
2847
|
-
this.addNewColumns(newFields);
|
|
1872
|
+
await this.addNewColumns(newFields);
|
|
2848
1873
|
}
|
|
2849
1874
|
}
|
|
2850
1875
|
}
|
|
@@ -2882,17 +1907,17 @@ const GoogleBigQuery = (function() {
|
|
|
2882
1907
|
return columns;
|
|
2883
1908
|
}
|
|
2884
1909
|
//---- createDatasetIfItDoesntExist --------------------------------
|
|
2885
|
-
createDatasetIfItDoesntExist() {
|
|
1910
|
+
async createDatasetIfItDoesntExist() {
|
|
2886
1911
|
let query = `---- Create Dataset if it not exists -----
|
|
2887
1912
|
`;
|
|
2888
1913
|
query += `CREATE SCHEMA IF NOT EXISTS \`${this.config.DestinationProjectID.value}.${this.config.DestinationDatasetName.value}\`
|
|
2889
1914
|
OPTIONS (
|
|
2890
1915
|
location = '${this.config.DestinationLocation.value}'
|
|
2891
1916
|
)`;
|
|
2892
|
-
this.executeQuery(query);
|
|
1917
|
+
await this.executeQuery(query);
|
|
2893
1918
|
}
|
|
2894
1919
|
//---- createTableIfItDoesntExist ----------------------------------
|
|
2895
|
-
createTableIfItDoesntExist() {
|
|
1920
|
+
async createTableIfItDoesntExist() {
|
|
2896
1921
|
let columns = [];
|
|
2897
1922
|
let columnPartitioned = null;
|
|
2898
1923
|
let existingColumns = {};
|
|
@@ -2928,15 +1953,14 @@ PARTITION BY ${columnPartitioned}`;
|
|
|
2928
1953
|
query += `
|
|
2929
1954
|
OPTIONS(description="${this.description}")`;
|
|
2930
1955
|
}
|
|
2931
|
-
this.executeQuery(query);
|
|
1956
|
+
await this.executeQuery(query);
|
|
2932
1957
|
this.config.logMessage(`Table ${this.config.DestinationDatasetID.value}.${this.config.DestinationTableName.value} was created`);
|
|
2933
1958
|
return existingColumns;
|
|
2934
1959
|
}
|
|
2935
1960
|
//---- checkIfGoogleBigQueryIsConnected ---------------------
|
|
2936
1961
|
checkIfGoogleBigQueryIsConnected() {
|
|
2937
1962
|
if (typeof BigQuery == "undefined") {
|
|
2938
|
-
throw new Error(`
|
|
2939
|
-
Extension / Apps Script / Editor / Services / + BigQuery API`);
|
|
1963
|
+
throw new Error(`BigQuery client library is not available. Ensure @google-cloud/bigquery is installed.`);
|
|
2940
1964
|
}
|
|
2941
1965
|
}
|
|
2942
1966
|
//---- addNewColumns -----------------------------------------------
|
|
@@ -2947,7 +1971,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
2947
1971
|
* @param {newColumns} array with a list of new columns
|
|
2948
1972
|
*
|
|
2949
1973
|
*/
|
|
2950
|
-
addNewColumns(newColumns) {
|
|
1974
|
+
async addNewColumns(newColumns) {
|
|
2951
1975
|
let query = "";
|
|
2952
1976
|
let columns = [];
|
|
2953
1977
|
for (var i in newColumns) {
|
|
@@ -2969,7 +1993,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
2969
1993
|
|
|
2970
1994
|
`;
|
|
2971
1995
|
query += columns.join(",\n");
|
|
2972
|
-
this.executeQuery(query);
|
|
1996
|
+
await this.executeQuery(query);
|
|
2973
1997
|
this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.DestinationDatasetID.value} dataset`);
|
|
2974
1998
|
}
|
|
2975
1999
|
}
|
|
@@ -2978,17 +2002,17 @@ OPTIONS(description="${this.description}")`;
|
|
|
2978
2002
|
* Saving data to a storage
|
|
2979
2003
|
* @param {data} array of assoc objects with records to save
|
|
2980
2004
|
*/
|
|
2981
|
-
saveData(data) {
|
|
2982
|
-
|
|
2005
|
+
async saveData(data) {
|
|
2006
|
+
for (const row of data) {
|
|
2983
2007
|
let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2984
2008
|
if (newFields.length > 0) {
|
|
2985
2009
|
console.log(newFields);
|
|
2986
|
-
this.addNewColumns(newFields);
|
|
2010
|
+
await this.addNewColumns(newFields);
|
|
2987
2011
|
}
|
|
2988
2012
|
this.addRecordToBuffer(row);
|
|
2989
|
-
this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2990
|
-
}
|
|
2991
|
-
this.saveRecordsAddedToBuffer();
|
|
2013
|
+
await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2014
|
+
}
|
|
2015
|
+
await this.saveRecordsAddedToBuffer();
|
|
2992
2016
|
}
|
|
2993
2017
|
// ------- addReordTuBuffer ---------------------
|
|
2994
2018
|
/**
|
|
@@ -3003,24 +2027,24 @@ OPTIONS(description="${this.description}")`;
|
|
|
3003
2027
|
* Add records from buffer to a sheet
|
|
3004
2028
|
* @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
|
|
3005
2029
|
*/
|
|
3006
|
-
saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2030
|
+
async saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
3007
2031
|
let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
|
|
3008
2032
|
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
3009
2033
|
console.log(`Starting BigQuery MERGE operation for ${bufferSize} records...`);
|
|
3010
|
-
this.executeQueryWithSizeLimit();
|
|
2034
|
+
await this.executeQueryWithSizeLimit();
|
|
3011
2035
|
}
|
|
3012
2036
|
}
|
|
3013
2037
|
//---- executeQueryWithSizeLimit ----------------------------------
|
|
3014
2038
|
/**
|
|
3015
2039
|
* Executes the MERGE query with automatic size reduction if it exceeds BigQuery limits
|
|
3016
2040
|
*/
|
|
3017
|
-
executeQueryWithSizeLimit() {
|
|
2041
|
+
async executeQueryWithSizeLimit() {
|
|
3018
2042
|
const bufferKeys = Object.keys(this.updatedRecordsBuffer);
|
|
3019
2043
|
const totalRecords = bufferKeys.length;
|
|
3020
2044
|
if (totalRecords === 0) {
|
|
3021
2045
|
return;
|
|
3022
2046
|
}
|
|
3023
|
-
this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
2047
|
+
await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
3024
2048
|
this.updatedRecordsBuffer = {};
|
|
3025
2049
|
}
|
|
3026
2050
|
//---- executeMergeQueryRecursively --------------------------------
|
|
@@ -3029,7 +2053,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3029
2053
|
* @param {Array} recordKeys - Array of record keys to process
|
|
3030
2054
|
* @param {number} batchSize - Current batch size to attempt
|
|
3031
2055
|
*/
|
|
3032
|
-
executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
2056
|
+
async executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
3033
2057
|
if (recordKeys.length === 0) {
|
|
3034
2058
|
return;
|
|
3035
2059
|
}
|
|
@@ -3043,11 +2067,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
3043
2067
|
const maxQuerySize = 1024 * 1024;
|
|
3044
2068
|
if (querySize > maxQuerySize) {
|
|
3045
2069
|
console.log(`Query size (${Math.round(querySize / 1024)}KB) exceeds BigQuery limit. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
|
|
3046
|
-
this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2070
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
3047
2071
|
return;
|
|
3048
2072
|
}
|
|
3049
2073
|
try {
|
|
3050
|
-
this.executeQuery(query);
|
|
2074
|
+
await this.executeQuery(query);
|
|
3051
2075
|
this.totalRecordsProcessed += currentBatch.length;
|
|
3052
2076
|
console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
|
|
3053
2077
|
if (remainingRecords.length > 0) {
|
|
@@ -3081,9 +2105,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
3081
2105
|
if (record[columnName] === void 0 || record[columnName] === null) {
|
|
3082
2106
|
columnValue = null;
|
|
3083
2107
|
} else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
|
|
3084
|
-
columnValue =
|
|
2108
|
+
columnValue = DateUtils3.formatDate(record[columnName]);
|
|
3085
2109
|
} else if (columnType.toUpperCase() == "DATETIME" && record[columnName] instanceof Date) {
|
|
3086
|
-
|
|
2110
|
+
const isoString = record[columnName].toISOString();
|
|
2111
|
+
columnValue = isoString.replace("T", " ").substring(0, 19);
|
|
3087
2112
|
} else {
|
|
3088
2113
|
columnValue = this.obfuscateSpecialCharacters(record[columnName]);
|
|
3089
2114
|
}
|
|
@@ -3118,53 +2143,36 @@ OPTIONS(description="${this.description}")`;
|
|
|
3118
2143
|
//---- query -------------------------------------------------------
|
|
3119
2144
|
/**
|
|
3120
2145
|
* Executes Google BigQuery Query and returns a result
|
|
3121
|
-
*
|
|
3122
|
-
* @param {query} string
|
|
3123
|
-
*
|
|
3124
|
-
* @return object
|
|
3125
|
-
*
|
|
2146
|
+
*
|
|
2147
|
+
* @param {query} string
|
|
2148
|
+
*
|
|
2149
|
+
* @return Promise<object>
|
|
2150
|
+
*
|
|
3126
2151
|
*/
|
|
3127
|
-
executeQuery(query) {
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
let error = void 0;
|
|
3137
|
-
let bigqueryClient = null;
|
|
3138
|
-
if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
|
|
3139
|
-
const { JWT } = require("google-auth-library");
|
|
3140
|
-
const credentials = JSON.parse(this.config.ServiceAccountJson.value);
|
|
3141
|
-
const authClient = new JWT({
|
|
3142
|
-
email: credentials.client_email,
|
|
3143
|
-
key: credentials.private_key,
|
|
3144
|
-
scopes: ["https://www.googleapis.com/auth/bigquery"]
|
|
3145
|
-
});
|
|
3146
|
-
bigqueryClient = new BigQuery({
|
|
3147
|
-
projectId: this.config.ProjectID.value || credentials.project_id,
|
|
3148
|
-
authClient
|
|
3149
|
-
});
|
|
3150
|
-
} else {
|
|
3151
|
-
throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
|
|
3152
|
-
}
|
|
3153
|
-
const options = {
|
|
3154
|
-
query,
|
|
3155
|
-
useLegacySql: false
|
|
3156
|
-
};
|
|
3157
|
-
bigqueryClient.createQueryJob(options).then(([job]) => job.getQueryResults()).then(([rows]) => rows).then((value) => {
|
|
3158
|
-
result = value;
|
|
3159
|
-
}).catch((e) => {
|
|
3160
|
-
error = e;
|
|
2152
|
+
async executeQuery(query) {
|
|
2153
|
+
let bigqueryClient = null;
|
|
2154
|
+
if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
|
|
2155
|
+
const { JWT } = require("google-auth-library");
|
|
2156
|
+
const credentials = JSON.parse(this.config.ServiceAccountJson.value);
|
|
2157
|
+
const authClient = new JWT({
|
|
2158
|
+
email: credentials.client_email,
|
|
2159
|
+
key: credentials.private_key,
|
|
2160
|
+
scopes: ["https://www.googleapis.com/auth/bigquery"]
|
|
3161
2161
|
});
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
}
|
|
3166
|
-
|
|
2162
|
+
bigqueryClient = new BigQuery({
|
|
2163
|
+
projectId: this.config.ProjectID.value || credentials.project_id,
|
|
2164
|
+
authClient
|
|
2165
|
+
});
|
|
2166
|
+
} else {
|
|
2167
|
+
throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
|
|
3167
2168
|
}
|
|
2169
|
+
const options = {
|
|
2170
|
+
query,
|
|
2171
|
+
useLegacySql: false
|
|
2172
|
+
};
|
|
2173
|
+
const [job] = await bigqueryClient.createQueryJob(options);
|
|
2174
|
+
const [rows] = await job.getQueryResults();
|
|
2175
|
+
return rows;
|
|
3168
2176
|
}
|
|
3169
2177
|
//---- obfuscateSpecialCharacters ----------------------------------
|
|
3170
2178
|
obfuscateSpecialCharacters(inputString) {
|
|
@@ -3260,7 +2268,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3260
2268
|
};
|
|
3261
2269
|
})();
|
|
3262
2270
|
const AwsAthena = (function() {
|
|
3263
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
2271
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
3264
2272
|
var AwsAthenaStorage = class AwsAthenaStorage extends AbstractStorage2 {
|
|
3265
2273
|
//---- constructor -------------------------------------------------
|
|
3266
2274
|
/**
|
|
@@ -3318,9 +2326,21 @@ const AwsAthena = (function() {
|
|
|
3318
2326
|
this.initAWS();
|
|
3319
2327
|
this.updatedRecordsBuffer = {};
|
|
3320
2328
|
this.existingColumns = {};
|
|
3321
|
-
this.setupAthenaDatabase();
|
|
3322
2329
|
this.uploadSid = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:.]/g, "") + "_" + Math.random().toString(36).substring(2, 15);
|
|
3323
2330
|
}
|
|
2331
|
+
//---- init --------------------------------------------------------
|
|
2332
|
+
/**
|
|
2333
|
+
* Initializing storage
|
|
2334
|
+
*/
|
|
2335
|
+
async init() {
|
|
2336
|
+
const success = await this.setupAthenaDatabase();
|
|
2337
|
+
if (success) {
|
|
2338
|
+
console.log("Database created or already exists");
|
|
2339
|
+
} else {
|
|
2340
|
+
throw new Error("Failed to create database");
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
//----------------------------------------------------------------
|
|
3324
2344
|
//---- initAWS ----------------------------------------------------
|
|
3325
2345
|
/**
|
|
3326
2346
|
* Initialize AWS SDK clients
|
|
@@ -3352,61 +2372,58 @@ const AwsAthena = (function() {
|
|
|
3352
2372
|
/**
|
|
3353
2373
|
* Create Athena database if it doesn't exist
|
|
3354
2374
|
*/
|
|
3355
|
-
createDatabaseIfNotExists() {
|
|
2375
|
+
async createDatabaseIfNotExists() {
|
|
3356
2376
|
const params = {
|
|
3357
2377
|
QueryString: `CREATE SCHEMA IF NOT EXISTS \`${this.config.AthenaDatabaseName.value}\``,
|
|
3358
2378
|
ResultConfiguration: {
|
|
3359
2379
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3360
2380
|
}
|
|
3361
2381
|
};
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
});
|
|
2382
|
+
await this.executeQuery(params, "ddl");
|
|
2383
|
+
this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
|
|
2384
|
+
return true;
|
|
3366
2385
|
}
|
|
3367
2386
|
//---- checkTableExists ------------------------------------------
|
|
3368
2387
|
/**
|
|
3369
2388
|
* Check if the target table exists in Athena
|
|
3370
2389
|
*/
|
|
3371
|
-
checkTableExists() {
|
|
2390
|
+
async checkTableExists() {
|
|
3372
2391
|
const params = {
|
|
3373
2392
|
QueryString: `SHOW TABLES IN \`${this.config.AthenaDatabaseName.value}\` LIKE '${this.config.DestinationTableName.value}'`,
|
|
3374
2393
|
ResultConfiguration: {
|
|
3375
2394
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3376
2395
|
}
|
|
3377
2396
|
};
|
|
3378
|
-
|
|
2397
|
+
try {
|
|
2398
|
+
const results = await this.executeQuery(params, "ddl");
|
|
3379
2399
|
if (results && results.length > 0) {
|
|
3380
|
-
return this.getTableSchema();
|
|
2400
|
+
return await this.getTableSchema();
|
|
3381
2401
|
}
|
|
3382
|
-
return this.createTargetTable();
|
|
3383
|
-
}
|
|
3384
|
-
return this.createTargetTable();
|
|
3385
|
-
}
|
|
2402
|
+
return await this.createTargetTable();
|
|
2403
|
+
} catch {
|
|
2404
|
+
return await this.createTargetTable();
|
|
2405
|
+
}
|
|
3386
2406
|
}
|
|
3387
2407
|
//---- getTableSchema -------------------------------------------
|
|
3388
2408
|
/**
|
|
3389
2409
|
* Get the schema of the existing table
|
|
3390
2410
|
*/
|
|
3391
|
-
getTableSchema() {
|
|
2411
|
+
async getTableSchema() {
|
|
3392
2412
|
const params = {
|
|
3393
2413
|
QueryString: `SHOW COLUMNS IN \`${this.config.AthenaDatabaseName.value}\`.\`${this.config.DestinationTableName.value}\``,
|
|
3394
2414
|
ResultConfiguration: {
|
|
3395
2415
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3396
2416
|
}
|
|
3397
2417
|
};
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
}).catch((error) => {
|
|
3408
|
-
return {};
|
|
3409
|
-
});
|
|
2418
|
+
const results = await this.executeQuery(params, "ddl");
|
|
2419
|
+
let columns = {};
|
|
2420
|
+
if (results && results.length > 0) {
|
|
2421
|
+
results.forEach((row) => {
|
|
2422
|
+
columns[row] = this.getColumnType(row);
|
|
2423
|
+
});
|
|
2424
|
+
}
|
|
2425
|
+
this.existingColumns = columns;
|
|
2426
|
+
return columns;
|
|
3410
2427
|
}
|
|
3411
2428
|
//---- createTargetTable ----------------------------------------------
|
|
3412
2429
|
/**
|
|
@@ -3496,45 +2513,42 @@ const AwsAthena = (function() {
|
|
|
3496
2513
|
* @param {Array} data - Array of objects with records to save
|
|
3497
2514
|
* @returns {Promise}
|
|
3498
2515
|
*/
|
|
3499
|
-
saveData(data) {
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
});
|
|
3518
|
-
}
|
|
2516
|
+
async saveData(data) {
|
|
2517
|
+
await this.checkTableExists();
|
|
2518
|
+
const allColumns = /* @__PURE__ */ new Set();
|
|
2519
|
+
if (data.length > 0) {
|
|
2520
|
+
data.forEach((row) => {
|
|
2521
|
+
Object.keys(row).forEach((column) => allColumns.add(column));
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
if (this.config.Fields.value) {
|
|
2525
|
+
this.getSelectedFields().forEach((columnName) => {
|
|
2526
|
+
if (columnName && !allColumns.has(columnName)) {
|
|
2527
|
+
allColumns.add(columnName);
|
|
2528
|
+
if (data.length > 0) {
|
|
2529
|
+
data.forEach((row) => {
|
|
2530
|
+
if (!row[columnName]) {
|
|
2531
|
+
row[columnName] = "";
|
|
2532
|
+
}
|
|
2533
|
+
});
|
|
3519
2534
|
}
|
|
3520
|
-
}
|
|
3521
|
-
}
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
deasync.loopWhile(() => !done);
|
|
2535
|
+
}
|
|
2536
|
+
});
|
|
2537
|
+
}
|
|
2538
|
+
const existingColumnsSet = new Set(Object.keys(this.existingColumns));
|
|
2539
|
+
const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
|
|
2540
|
+
if (newColumns.length > 0) {
|
|
2541
|
+
await this.addNewColumns(newColumns);
|
|
2542
|
+
}
|
|
2543
|
+
if (data.length === 0) {
|
|
2544
|
+
return;
|
|
2545
|
+
}
|
|
2546
|
+
this.config.logMessage(`Saving ${data.length} records to Athena`);
|
|
2547
|
+
const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
|
|
2548
|
+
await this.uploadDataToS3TempFolder(data, tempFolder);
|
|
2549
|
+
const tempTableName = await this.createTempTable(tempFolder, this.uploadSid);
|
|
2550
|
+
await this.mergeDataFromTempTable(tempTableName, this.uploadSid);
|
|
2551
|
+
await this.cleanupTempResources(tempFolder, tempTableName);
|
|
3538
2552
|
}
|
|
3539
2553
|
//---- uploadDataToS3TempFolder ---------------------------------
|
|
3540
2554
|
/**
|
|
@@ -3723,6 +2737,7 @@ const AwsAthena = (function() {
|
|
|
3723
2737
|
QueryExecutionId: queryExecutionId
|
|
3724
2738
|
};
|
|
3725
2739
|
return this.athenaClient.send(new GetQueryExecutionCommand(params)).then((data) => {
|
|
2740
|
+
var _a;
|
|
3726
2741
|
const state = data.QueryExecution.Status.State;
|
|
3727
2742
|
if (state === "SUCCEEDED") {
|
|
3728
2743
|
return new Promise((resolve) => {
|
|
@@ -3731,6 +2746,7 @@ const AwsAthena = (function() {
|
|
|
3731
2746
|
}, 3e3);
|
|
3732
2747
|
});
|
|
3733
2748
|
} else if (state === "FAILED" || state === "CANCELLED") {
|
|
2749
|
+
this.config.logMessage(`Query ${queryExecutionId} ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}. Error: ${((_a = data.QueryExecution.Status.Error) == null ? void 0 : _a.Message) || ""}`);
|
|
3734
2750
|
throw new Error(`Query ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}`);
|
|
3735
2751
|
} else {
|
|
3736
2752
|
return new Promise((resolve) => {
|
|
@@ -3890,12 +2906,11 @@ const AwsAthena = (function() {
|
|
|
3890
2906
|
};
|
|
3891
2907
|
})();
|
|
3892
2908
|
const Storages = {
|
|
3893
|
-
GoogleSheets,
|
|
3894
2909
|
GoogleBigQuery,
|
|
3895
2910
|
AwsAthena
|
|
3896
2911
|
};
|
|
3897
2912
|
const XAds = (function() {
|
|
3898
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
2913
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
3899
2914
|
const XAdsHelper = {
|
|
3900
2915
|
/**
|
|
3901
2916
|
* Parse fields string into a structured object
|
|
@@ -4789,34 +3804,34 @@ const XAds = (function() {
|
|
|
4789
3804
|
* @param {string} [opts.end_time]
|
|
4790
3805
|
* @returns {Array<Object>}
|
|
4791
3806
|
*/
|
|
4792
|
-
fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
|
|
4793
|
-
|
|
3807
|
+
async fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
|
|
3808
|
+
await AsyncUtils3.delay(this.config.AdsApiDelay.value * 1e3);
|
|
4794
3809
|
switch (nodeName) {
|
|
4795
3810
|
case "accounts": {
|
|
4796
|
-
const resp = this._getData(`accounts/${accountId}`, "accounts", fields);
|
|
3811
|
+
const resp = await this._getData(`accounts/${accountId}`, "accounts", fields);
|
|
4797
3812
|
return [resp.data];
|
|
4798
3813
|
}
|
|
4799
3814
|
case "campaigns":
|
|
4800
3815
|
case "line_items":
|
|
4801
3816
|
case "promoted_tweets":
|
|
4802
3817
|
case "tweets":
|
|
4803
|
-
return this._catalogFetch({
|
|
3818
|
+
return await this._catalogFetch({
|
|
4804
3819
|
nodeName,
|
|
4805
3820
|
accountId,
|
|
4806
3821
|
fields,
|
|
4807
3822
|
pageSize: this.config.DataMaxCount.value
|
|
4808
3823
|
});
|
|
4809
3824
|
case "cards":
|
|
4810
|
-
return this._catalogFetch({
|
|
3825
|
+
return await this._catalogFetch({
|
|
4811
3826
|
nodeName,
|
|
4812
3827
|
accountId,
|
|
4813
3828
|
fields,
|
|
4814
3829
|
pageSize: this.config.CardsMaxCountPerRequest.value
|
|
4815
3830
|
});
|
|
4816
3831
|
case "cards_all":
|
|
4817
|
-
return this._fetchAllCards(accountId, fields);
|
|
3832
|
+
return await this._fetchAllCards(accountId, fields);
|
|
4818
3833
|
case "stats":
|
|
4819
|
-
return this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
|
|
3834
|
+
return await this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
|
|
4820
3835
|
default:
|
|
4821
3836
|
throw new ConfigurationError(`Unknown node: ${nodeName}`);
|
|
4822
3837
|
}
|
|
@@ -4852,7 +3867,7 @@ const XAds = (function() {
|
|
|
4852
3867
|
/**
|
|
4853
3868
|
* Shared logic for non-time-series endpoints
|
|
4854
3869
|
*/
|
|
4855
|
-
_catalogFetch({ nodeName, accountId, fields, pageSize }) {
|
|
3870
|
+
async _catalogFetch({ nodeName, accountId, fields, pageSize }) {
|
|
4856
3871
|
const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
|
|
4857
3872
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
4858
3873
|
if (missingKeys.length > 0) {
|
|
@@ -4876,7 +3891,7 @@ const XAds = (function() {
|
|
|
4876
3891
|
console.log("deleting cached tweets");
|
|
4877
3892
|
this._tweetsCache.delete(accountId);
|
|
4878
3893
|
}
|
|
4879
|
-
let all = this._fetchPages({
|
|
3894
|
+
let all = await this._fetchPages({
|
|
4880
3895
|
accountId,
|
|
4881
3896
|
nodeName,
|
|
4882
3897
|
fields,
|
|
@@ -4900,7 +3915,7 @@ const XAds = (function() {
|
|
|
4900
3915
|
/**
|
|
4901
3916
|
* Shared pagination logic
|
|
4902
3917
|
*/
|
|
4903
|
-
_fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
|
|
3918
|
+
async _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
|
|
4904
3919
|
const all = [];
|
|
4905
3920
|
let cursor = null;
|
|
4906
3921
|
const MAX_PAGES = 100;
|
|
@@ -4911,7 +3926,7 @@ const XAds = (function() {
|
|
|
4911
3926
|
...extraParams,
|
|
4912
3927
|
...cursor ? { cursor } : {}
|
|
4913
3928
|
};
|
|
4914
|
-
const resp = this._getData(
|
|
3929
|
+
const resp = await this._getData(
|
|
4915
3930
|
`accounts/${accountId}/${nodeName}`,
|
|
4916
3931
|
nodeName,
|
|
4917
3932
|
fields,
|
|
@@ -4932,15 +3947,15 @@ const XAds = (function() {
|
|
|
4932
3947
|
* Fetch all cards by first collecting URIs from tweets,
|
|
4933
3948
|
* then calling the cards/all endpoint in chunks.
|
|
4934
3949
|
*/
|
|
4935
|
-
_fetchAllCards(accountId, fields) {
|
|
4936
|
-
const tweets = this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
|
|
3950
|
+
async _fetchAllCards(accountId, fields) {
|
|
3951
|
+
const tweets = await this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
|
|
4937
3952
|
const uris = tweets.map((t) => t.card_uri).filter(Boolean);
|
|
4938
3953
|
if (!uris.length) return [];
|
|
4939
3954
|
const all = [];
|
|
4940
3955
|
const chunkSize = this.config.CardsMaxCountPerRequest.value;
|
|
4941
3956
|
for (let i = 0; i < uris.length; i += chunkSize) {
|
|
4942
3957
|
const chunk = uris.slice(i, i + chunkSize);
|
|
4943
|
-
const resp = this._getData(
|
|
3958
|
+
const resp = await this._getData(
|
|
4944
3959
|
`accounts/${accountId}/cards/all`,
|
|
4945
3960
|
"cards_all",
|
|
4946
3961
|
fields,
|
|
@@ -4957,18 +3972,18 @@ const XAds = (function() {
|
|
|
4957
3972
|
/**
|
|
4958
3973
|
* Stats are time-series and need flattening of `metrics`
|
|
4959
3974
|
*/
|
|
4960
|
-
_timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
|
|
3975
|
+
async _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
|
|
4961
3976
|
const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
|
|
4962
3977
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
4963
3978
|
if (missingKeys.length > 0) {
|
|
4964
3979
|
throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
|
|
4965
3980
|
}
|
|
4966
|
-
const promos = this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
|
|
3981
|
+
const promos = await this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
|
|
4967
3982
|
const ids = promos.map((r) => r.id);
|
|
4968
3983
|
if (!ids.length) return [];
|
|
4969
3984
|
const e = new Date(end_time);
|
|
4970
3985
|
e.setDate(e.getDate() + 1);
|
|
4971
|
-
const endStr =
|
|
3986
|
+
const endStr = DateUtils3.formatDate(e);
|
|
4972
3987
|
const result = [];
|
|
4973
3988
|
for (let i = 0; i < ids.length; i += this.config.StatsMaxEntityIds.value) {
|
|
4974
3989
|
const batch = ids.slice(i, i + this.config.StatsMaxEntityIds.value).join(",");
|
|
@@ -4981,7 +3996,7 @@ const XAds = (function() {
|
|
|
4981
3996
|
end_time: endStr
|
|
4982
3997
|
};
|
|
4983
3998
|
for (const placement of ["ALL_ON_TWITTER", "PUBLISHER_NETWORK"]) {
|
|
4984
|
-
const raw = this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
|
|
3999
|
+
const raw = await this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
|
|
4985
4000
|
const arr = Array.isArray(raw.data) ? raw.data : [raw.data];
|
|
4986
4001
|
arr.forEach((h) => {
|
|
4987
4002
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
|
|
@@ -5017,18 +4032,19 @@ const XAds = (function() {
|
|
|
5017
4032
|
/**
|
|
5018
4033
|
* Pull JSON from the Ads API (raw, no field-filter).
|
|
5019
4034
|
*/
|
|
5020
|
-
_rawFetch(path, params = {}) {
|
|
4035
|
+
async _rawFetch(path, params = {}) {
|
|
5021
4036
|
const url = `${this.BASE_URL}${this.config.Version.value}/${path}`;
|
|
5022
4037
|
const qs = Object.keys(params).length ? "?" + Object.entries(params).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
|
|
5023
4038
|
const finalUrl = url + qs;
|
|
5024
4039
|
const oauth = this._generateOAuthHeader({ method: "GET", url, params });
|
|
5025
|
-
|
|
5026
|
-
const resp = this.urlFetchWithRetry(finalUrl, {
|
|
4040
|
+
await AsyncUtils3.delay(1e3);
|
|
4041
|
+
const resp = await this.urlFetchWithRetry(finalUrl, {
|
|
5027
4042
|
method: "GET",
|
|
5028
4043
|
headers: { Authorization: oauth, "Content-Type": "application/json" },
|
|
5029
4044
|
muteHttpExceptions: true
|
|
5030
4045
|
});
|
|
5031
|
-
|
|
4046
|
+
const text = await resp.getContentText();
|
|
4047
|
+
return JSON.parse(text);
|
|
5032
4048
|
}
|
|
5033
4049
|
/**
|
|
5034
4050
|
* Determines if a X Ads API error is valid for retry
|
|
@@ -5051,8 +4067,8 @@ const XAds = (function() {
|
|
|
5051
4067
|
}
|
|
5052
4068
|
return false;
|
|
5053
4069
|
}
|
|
5054
|
-
_getData(path, nodeName, fields, extraParams = {}) {
|
|
5055
|
-
const json = this._rawFetch(path, extraParams);
|
|
4070
|
+
async _getData(path, nodeName, fields, extraParams = {}) {
|
|
4071
|
+
const json = await this._rawFetch(path, extraParams);
|
|
5056
4072
|
if (!json.data) return json;
|
|
5057
4073
|
const arr = Array.isArray(json.data) ? json.data : [json.data];
|
|
5058
4074
|
const filtered = this._filterBySchema(arr, nodeName, fields);
|
|
@@ -5091,7 +4107,7 @@ const XAds = (function() {
|
|
|
5091
4107
|
const { ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret } = this.config;
|
|
5092
4108
|
const oauth = {
|
|
5093
4109
|
oauth_consumer_key: ConsumerKey.value,
|
|
5094
|
-
oauth_nonce:
|
|
4110
|
+
oauth_nonce: CryptoUtils3.getUuid().replace(/-/g, ""),
|
|
5095
4111
|
oauth_signature_method: "HMAC-SHA1",
|
|
5096
4112
|
oauth_timestamp: Math.floor(Date.now() / 1e3),
|
|
5097
4113
|
oauth_token: AccessToken.value,
|
|
@@ -5106,9 +4122,9 @@ const XAds = (function() {
|
|
|
5106
4122
|
)
|
|
5107
4123
|
].join("&");
|
|
5108
4124
|
const signingKey = encodeURIComponent(ConsumerSecret.value) + "&" + encodeURIComponent(AccessTokenSecret.value);
|
|
5109
|
-
oauth.oauth_signature =
|
|
5110
|
-
|
|
5111
|
-
|
|
4125
|
+
oauth.oauth_signature = CryptoUtils3.base64Encode(
|
|
4126
|
+
CryptoUtils3.computeHmacSignature(
|
|
4127
|
+
CryptoUtils3.MacAlgorithm.HMAC_SHA_1,
|
|
5112
4128
|
baseString,
|
|
5113
4129
|
signingKey
|
|
5114
4130
|
)
|
|
@@ -5124,7 +4140,7 @@ const XAds = (function() {
|
|
|
5124
4140
|
}
|
|
5125
4141
|
};
|
|
5126
4142
|
var XAdsConnector = class XAdsConnector extends AbstractConnector3 {
|
|
5127
|
-
constructor(config, source, storageName = "
|
|
4143
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
5128
4144
|
super(config, source, null, runConfig);
|
|
5129
4145
|
this.storageName = storageName;
|
|
5130
4146
|
}
|
|
@@ -5132,12 +4148,12 @@ const XAds = (function() {
|
|
|
5132
4148
|
* Main method - entry point for the import process
|
|
5133
4149
|
* Processes all nodes defined in the fields configuration
|
|
5134
4150
|
*/
|
|
5135
|
-
startImportProcess() {
|
|
4151
|
+
async startImportProcess() {
|
|
5136
4152
|
const fields = XAdsHelper.parseFields(this.config.Fields.value);
|
|
5137
4153
|
const accountIds = XAdsHelper.parseAccountIds(this.config.AccountIDs.value);
|
|
5138
4154
|
for (const accountId of accountIds) {
|
|
5139
4155
|
for (const nodeName in fields) {
|
|
5140
|
-
this.processNode({
|
|
4156
|
+
await this.processNode({
|
|
5141
4157
|
nodeName,
|
|
5142
4158
|
accountId,
|
|
5143
4159
|
fields: fields[nodeName] || []
|
|
@@ -5153,15 +4169,15 @@ const XAds = (function() {
|
|
|
5153
4169
|
* @param {string} options.accountId - Account ID
|
|
5154
4170
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5155
4171
|
*/
|
|
5156
|
-
processNode({ nodeName, accountId, fields }) {
|
|
4172
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
5157
4173
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
5158
|
-
this.processTimeSeriesNode({
|
|
4174
|
+
await this.processTimeSeriesNode({
|
|
5159
4175
|
nodeName,
|
|
5160
4176
|
accountId,
|
|
5161
4177
|
fields
|
|
5162
4178
|
});
|
|
5163
4179
|
} else {
|
|
5164
|
-
this.processCatalogNode({
|
|
4180
|
+
await this.processCatalogNode({
|
|
5165
4181
|
nodeName,
|
|
5166
4182
|
accountId,
|
|
5167
4183
|
fields
|
|
@@ -5176,7 +4192,7 @@ const XAds = (function() {
|
|
|
5176
4192
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5177
4193
|
* @param {Object} options.storage - Storage instance
|
|
5178
4194
|
*/
|
|
5179
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
4195
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
5180
4196
|
var _a;
|
|
5181
4197
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
5182
4198
|
if (daysToFetch <= 0) {
|
|
@@ -5186,12 +4202,13 @@ const XAds = (function() {
|
|
|
5186
4202
|
for (let i = 0; i < daysToFetch; i++) {
|
|
5187
4203
|
const currentDate = new Date(startDate);
|
|
5188
4204
|
currentDate.setDate(currentDate.getDate() + i);
|
|
5189
|
-
const formattedDate =
|
|
5190
|
-
const data = this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
|
|
4205
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
4206
|
+
const data = await this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
|
|
5191
4207
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
|
|
5192
4208
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
5193
4209
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
5194
|
-
this.getStorageByNode(nodeName)
|
|
4210
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
4211
|
+
await storage.saveData(preparedData);
|
|
5195
4212
|
}
|
|
5196
4213
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
5197
4214
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -5206,13 +4223,14 @@ const XAds = (function() {
|
|
|
5206
4223
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5207
4224
|
* @param {Object} options.storage - Storage instance
|
|
5208
4225
|
*/
|
|
5209
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
4226
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
5210
4227
|
var _a;
|
|
5211
|
-
const data = this.source.fetchData({ nodeName, accountId, fields });
|
|
4228
|
+
const data = await this.source.fetchData({ nodeName, accountId, fields });
|
|
5212
4229
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
|
|
5213
4230
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
5214
4231
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
5215
|
-
this.getStorageByNode(nodeName)
|
|
4232
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
4233
|
+
await storage.saveData(preparedData);
|
|
5216
4234
|
}
|
|
5217
4235
|
}
|
|
5218
4236
|
/**
|
|
@@ -5220,7 +4238,7 @@ const XAds = (function() {
|
|
|
5220
4238
|
* @param {string} nodeName - Name of the node
|
|
5221
4239
|
* @returns {Object} Storage instance
|
|
5222
4240
|
*/
|
|
5223
|
-
getStorageByNode(nodeName) {
|
|
4241
|
+
async getStorageByNode(nodeName) {
|
|
5224
4242
|
if (!("storages" in this)) {
|
|
5225
4243
|
this.storages = {};
|
|
5226
4244
|
}
|
|
@@ -5238,6 +4256,7 @@ const XAds = (function() {
|
|
|
5238
4256
|
this.source.fieldsSchema[nodeName].fields,
|
|
5239
4257
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
5240
4258
|
);
|
|
4259
|
+
await this.storages[nodeName].init();
|
|
5241
4260
|
}
|
|
5242
4261
|
return this.storages[nodeName];
|
|
5243
4262
|
}
|
|
@@ -5254,7 +4273,7 @@ const XAds = (function() {
|
|
|
5254
4273
|
};
|
|
5255
4274
|
})();
|
|
5256
4275
|
const TikTokAds = (function() {
|
|
5257
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
4276
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
5258
4277
|
class TiktokMarketingApiProvider {
|
|
5259
4278
|
constructor(appId, accessToken, appSecret, isSandbox = false) {
|
|
5260
4279
|
this.BASE_URL = "https://business-api.tiktok.com/open_api/";
|
|
@@ -5278,7 +4297,7 @@ const TikTokAds = (function() {
|
|
|
5278
4297
|
getApiVersion() {
|
|
5279
4298
|
return this.API_VERSION;
|
|
5280
4299
|
}
|
|
5281
|
-
makeRequest(options) {
|
|
4300
|
+
async makeRequest(options) {
|
|
5282
4301
|
const { url, method, data } = options;
|
|
5283
4302
|
const headers = {
|
|
5284
4303
|
"Access-Token": this.accessToken,
|
|
@@ -5287,21 +4306,21 @@ const TikTokAds = (function() {
|
|
|
5287
4306
|
let backoff = this.INITIAL_BACKOFF;
|
|
5288
4307
|
for (let retries = 0; retries < this.MAX_RETRIES; retries++) {
|
|
5289
4308
|
try {
|
|
5290
|
-
const response =
|
|
4309
|
+
const response = await HttpUtils3.fetch(url, {
|
|
5291
4310
|
method,
|
|
5292
4311
|
headers,
|
|
5293
|
-
body: data ? JSON.stringify(data) : null
|
|
5294
|
-
muteHttpExceptions: true
|
|
4312
|
+
body: data ? JSON.stringify(data) : null
|
|
5295
4313
|
});
|
|
5296
4314
|
const responseCode = response.getResponseCode();
|
|
4315
|
+
const text = await response.getContentText();
|
|
5297
4316
|
if (responseCode !== this.SUCCESS_RESPONSE_CODE) {
|
|
5298
|
-
throw new Error(`TikTok API error: ${
|
|
4317
|
+
throw new Error(`TikTok API error: ${text}`);
|
|
5299
4318
|
}
|
|
5300
|
-
const jsonData = JSON.parse(
|
|
4319
|
+
const jsonData = JSON.parse(text);
|
|
5301
4320
|
if (jsonData.code !== this.SUCCESS_CODE) {
|
|
5302
4321
|
if (jsonData.code === this.RATE_LIMIT_CODE) {
|
|
5303
4322
|
console.error("TikTok Marketing API rate limit exceeded. Retrying...");
|
|
5304
|
-
|
|
4323
|
+
await AsyncUtils3.delay(backoff);
|
|
5305
4324
|
backoff *= 2;
|
|
5306
4325
|
continue;
|
|
5307
4326
|
}
|
|
@@ -5310,7 +4329,7 @@ const TikTokAds = (function() {
|
|
|
5310
4329
|
return jsonData;
|
|
5311
4330
|
} catch (error) {
|
|
5312
4331
|
if (retries < this.MAX_RETRIES - 1 && error.message.includes("rate limit")) {
|
|
5313
|
-
|
|
4332
|
+
await AsyncUtils3.delay(backoff);
|
|
5314
4333
|
backoff *= 2;
|
|
5315
4334
|
} else {
|
|
5316
4335
|
throw error;
|
|
@@ -5318,7 +4337,7 @@ const TikTokAds = (function() {
|
|
|
5318
4337
|
}
|
|
5319
4338
|
}
|
|
5320
4339
|
}
|
|
5321
|
-
handlePagination(endpoint, params = {}) {
|
|
4340
|
+
async handlePagination(endpoint, params = {}) {
|
|
5322
4341
|
let allData = [];
|
|
5323
4342
|
let page = 1;
|
|
5324
4343
|
let hasMorePages = true;
|
|
@@ -5326,7 +4345,7 @@ const TikTokAds = (function() {
|
|
|
5326
4345
|
while (hasMorePages) {
|
|
5327
4346
|
const paginatedParams = { ...params, page, page_size: pageSize };
|
|
5328
4347
|
const url = this.buildUrl(endpoint, paginatedParams);
|
|
5329
|
-
const response = this.makeRequest({ url, method: "GET" });
|
|
4348
|
+
const response = await this.makeRequest({ url, method: "GET" });
|
|
5330
4349
|
const pageData = response.data.list || [];
|
|
5331
4350
|
allData = allData.concat(pageData);
|
|
5332
4351
|
const total = response.data.page_info ? response.data.page_info.total_number : 0;
|
|
@@ -5334,7 +4353,7 @@ const TikTokAds = (function() {
|
|
|
5334
4353
|
hasMorePages = currentCount < total && pageData.length > 0;
|
|
5335
4354
|
page++;
|
|
5336
4355
|
if (hasMorePages) {
|
|
5337
|
-
|
|
4356
|
+
await AsyncUtils3.delay(100);
|
|
5338
4357
|
}
|
|
5339
4358
|
}
|
|
5340
4359
|
return allData;
|
|
@@ -6417,7 +5436,7 @@ const TikTokAds = (function() {
|
|
|
6417
5436
|
* @param {Date} endDate - End date for time-series data (optional)
|
|
6418
5437
|
* @return {array} - Array of data objects
|
|
6419
5438
|
*/
|
|
6420
|
-
fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
|
|
5439
|
+
async fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
|
|
6421
5440
|
if (!this.fieldsSchema[nodeName]) {
|
|
6422
5441
|
throw new Error(`Unknown node type: ${nodeName}`);
|
|
6423
5442
|
}
|
|
@@ -6438,8 +5457,8 @@ const TikTokAds = (function() {
|
|
|
6438
5457
|
let formattedStartDate = null;
|
|
6439
5458
|
let formattedEndDate = null;
|
|
6440
5459
|
if (startDate) {
|
|
6441
|
-
formattedStartDate =
|
|
6442
|
-
formattedEndDate = endDate ?
|
|
5460
|
+
formattedStartDate = DateUtils3.formatDate(startDate);
|
|
5461
|
+
formattedEndDate = endDate ? DateUtils3.formatDate(endDate) : formattedStartDate;
|
|
6443
5462
|
}
|
|
6444
5463
|
let filtering = null;
|
|
6445
5464
|
if (this.config.IncludeDeleted && this.config.IncludeDeleted.value) {
|
|
@@ -6661,11 +5680,11 @@ const TikTokAds = (function() {
|
|
|
6661
5680
|
}
|
|
6662
5681
|
};
|
|
6663
5682
|
var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector3 {
|
|
6664
|
-
constructor(config, source, storageName = "
|
|
5683
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
6665
5684
|
super(config, source, null, runConfig);
|
|
6666
5685
|
this.storageName = storageName;
|
|
6667
5686
|
}
|
|
6668
|
-
startImportProcess() {
|
|
5687
|
+
async startImportProcess() {
|
|
6669
5688
|
try {
|
|
6670
5689
|
let advertiserIds = TikTokAdsHelper.parseAdvertiserIds(this.config.AdvertiserIDs.value || "");
|
|
6671
5690
|
if (!advertiserIds || advertiserIds.length === 0) {
|
|
@@ -6689,7 +5708,7 @@ const TikTokAds = (function() {
|
|
|
6689
5708
|
}
|
|
6690
5709
|
}
|
|
6691
5710
|
if (Object.keys(catalogNodes).length > 0) {
|
|
6692
|
-
this.importCatalogData(catalogNodes, advertiserIds);
|
|
5711
|
+
await this.importCatalogData(catalogNodes, advertiserIds);
|
|
6693
5712
|
}
|
|
6694
5713
|
if (Object.keys(timeSeriesNodes).length > 0) {
|
|
6695
5714
|
try {
|
|
@@ -6698,7 +5717,7 @@ const TikTokAds = (function() {
|
|
|
6698
5717
|
this.config.logMessage("There is nothing to import in this data range");
|
|
6699
5718
|
return;
|
|
6700
5719
|
}
|
|
6701
|
-
this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
|
|
5720
|
+
await this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
|
|
6702
5721
|
} catch (error) {
|
|
6703
5722
|
this.config.logMessage(`Error determining date range: ${error.message}`);
|
|
6704
5723
|
console.error(error.stack);
|
|
@@ -6717,35 +5736,36 @@ const TikTokAds = (function() {
|
|
|
6717
5736
|
}
|
|
6718
5737
|
/**
|
|
6719
5738
|
* Imports all catalog (non-time-series) data types
|
|
6720
|
-
*
|
|
5739
|
+
*
|
|
6721
5740
|
* @param {object} catalogNodes - Object with node names as keys and field arrays as values
|
|
6722
5741
|
* @param {array} advertiserIds - List of advertiser IDs to fetch data for
|
|
6723
5742
|
*/
|
|
6724
|
-
importCatalogData(catalogNodes, advertiserIds) {
|
|
5743
|
+
async importCatalogData(catalogNodes, advertiserIds) {
|
|
6725
5744
|
for (var nodeName in catalogNodes) {
|
|
6726
5745
|
this.config.logMessage(`Starting import for ${nodeName} data...`);
|
|
6727
|
-
this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
|
|
5746
|
+
await this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
|
|
6728
5747
|
}
|
|
6729
5748
|
}
|
|
6730
5749
|
/**
|
|
6731
5750
|
* Imports catalog (not time series) data
|
|
6732
|
-
*
|
|
5751
|
+
*
|
|
6733
5752
|
* @param {string} nodeName - Node name
|
|
6734
5753
|
* @param {array} advertiserIds - List of advertiser IDs
|
|
6735
5754
|
* @param {array} fields - List of fields
|
|
6736
5755
|
*/
|
|
6737
|
-
startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
|
|
5756
|
+
async startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
|
|
6738
5757
|
var _a;
|
|
6739
5758
|
this.config.logMessage(`Fetching all available fields for ${nodeName}`);
|
|
6740
5759
|
for (var i in advertiserIds) {
|
|
6741
5760
|
let advertiserId = advertiserIds[i];
|
|
6742
5761
|
try {
|
|
6743
|
-
let data = this.source.fetchData(nodeName, advertiserId, fields);
|
|
5762
|
+
let data = await this.source.fetchData(nodeName, advertiserId, fields);
|
|
6744
5763
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for advertiser ${advertiserId}` : `No records have been fetched`);
|
|
6745
5764
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
6746
5765
|
try {
|
|
6747
5766
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
6748
|
-
this.getStorageByNode(nodeName)
|
|
5767
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
5768
|
+
await storage.saveData(preparedData);
|
|
6749
5769
|
} catch (storageError) {
|
|
6750
5770
|
this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
|
|
6751
5771
|
console.error(`Error details: ${storageError.stack}`);
|
|
@@ -6759,29 +5779,30 @@ const TikTokAds = (function() {
|
|
|
6759
5779
|
}
|
|
6760
5780
|
/**
|
|
6761
5781
|
* Imports time series data
|
|
6762
|
-
*
|
|
5782
|
+
*
|
|
6763
5783
|
* @param {array} advertiserIds - List of advertiser IDs
|
|
6764
5784
|
* @param {object} timeSeriesNodes - Object of properties, each is array of fields
|
|
6765
5785
|
* @param {Date} startDate - Start date
|
|
6766
5786
|
* @param {number} daysToFetch - Number of days to fetch
|
|
6767
5787
|
*/
|
|
6768
|
-
startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
|
|
5788
|
+
async startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
|
|
6769
5789
|
var _a;
|
|
6770
5790
|
for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
6771
5791
|
const currentDate = new Date(startDate);
|
|
6772
5792
|
currentDate.setDate(currentDate.getDate() + daysShift);
|
|
6773
|
-
const formattedDate =
|
|
5793
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
6774
5794
|
this.config.logMessage(`Processing data for date: ${formattedDate}`);
|
|
6775
5795
|
for (let advertiserId of advertiserIds) {
|
|
6776
5796
|
for (var nodeName in timeSeriesNodes) {
|
|
6777
5797
|
try {
|
|
6778
5798
|
this.config.logMessage(`Start importing data for ${formattedDate}: ${advertiserId}/${nodeName}`);
|
|
6779
|
-
let data = this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
|
|
5799
|
+
let data = await this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
|
|
6780
5800
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
6781
5801
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
6782
5802
|
try {
|
|
6783
5803
|
const preparedData = data.length ? this.addMissingFieldsToData(data, timeSeriesNodes[nodeName]) : data;
|
|
6784
|
-
this.getStorageByNode(nodeName)
|
|
5804
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
5805
|
+
await storage.saveData(preparedData);
|
|
6785
5806
|
} catch (storageError) {
|
|
6786
5807
|
this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
|
|
6787
5808
|
console.error(`Error details: ${storageError.stack}`);
|
|
@@ -6805,7 +5826,7 @@ const TikTokAds = (function() {
|
|
|
6805
5826
|
* @param {array} requestedFields - List of requested fields
|
|
6806
5827
|
* @return {AbstractStorage} - Storage instance
|
|
6807
5828
|
*/
|
|
6808
|
-
getStorageByNode(nodeName) {
|
|
5829
|
+
async getStorageByNode(nodeName) {
|
|
6809
5830
|
if (!("storages" in this)) {
|
|
6810
5831
|
this.storages = {};
|
|
6811
5832
|
}
|
|
@@ -6827,6 +5848,7 @@ const TikTokAds = (function() {
|
|
|
6827
5848
|
this.source.fieldsSchema[nodeName]["fields"] || {},
|
|
6828
5849
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
6829
5850
|
);
|
|
5851
|
+
await this.storages[nodeName].init();
|
|
6830
5852
|
}
|
|
6831
5853
|
return this.storages[nodeName];
|
|
6832
5854
|
}
|
|
@@ -6844,35 +5866,9 @@ const TikTokAds = (function() {
|
|
|
6844
5866
|
this.config.logMessage(`Cleaning up data older than ${keepDays} days...`);
|
|
6845
5867
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
6846
5868
|
cutoffDate.setDate(cutoffDate.getDate() - keepDays);
|
|
6847
|
-
|
|
5869
|
+
DateUtils3.formatDate(cutoffDate);
|
|
6848
5870
|
for (var nodeName in this.source.fieldsSchema) {
|
|
6849
|
-
if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"]))
|
|
6850
|
-
try {
|
|
6851
|
-
const storage = this.getStorageByNode(nodeName);
|
|
6852
|
-
if (storage instanceof GoogleSheetsStorage) {
|
|
6853
|
-
const dateField = this.source.fieldsSchema[nodeName]["fields"]["date_start"] ? "date_start" : "stat_time_day";
|
|
6854
|
-
let keysToDelete = [];
|
|
6855
|
-
for (const uniqueKey in storage.values) {
|
|
6856
|
-
const record = storage.values[uniqueKey];
|
|
6857
|
-
const rowDate = record[dateField];
|
|
6858
|
-
if (rowDate && rowDate < cutoffDate) {
|
|
6859
|
-
keysToDelete.push(uniqueKey);
|
|
6860
|
-
}
|
|
6861
|
-
}
|
|
6862
|
-
let deletedCount = 0;
|
|
6863
|
-
for (const key of keysToDelete) {
|
|
6864
|
-
storage.deleteRecord(key);
|
|
6865
|
-
deletedCount++;
|
|
6866
|
-
}
|
|
6867
|
-
if (deletedCount > 0) {
|
|
6868
|
-
this.config.logMessage(`Deleted ${deletedCount} rows from ${nodeName} that were older than ${formattedCutoffDate}`);
|
|
6869
|
-
}
|
|
6870
|
-
}
|
|
6871
|
-
} catch (error) {
|
|
6872
|
-
this.config.logMessage(`Error cleaning up old data from ${nodeName}: ${error.message}`);
|
|
6873
|
-
console.error(`Error details: ${error.stack}`);
|
|
6874
|
-
}
|
|
6875
|
-
}
|
|
5871
|
+
if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) ;
|
|
6876
5872
|
}
|
|
6877
5873
|
}
|
|
6878
5874
|
};
|
|
@@ -6889,7 +5885,7 @@ const TikTokAds = (function() {
|
|
|
6889
5885
|
};
|
|
6890
5886
|
})();
|
|
6891
5887
|
const RedditAds = (function() {
|
|
6892
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
5888
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
6893
5889
|
const RedditAdsHelper = {
|
|
6894
5890
|
/**
|
|
6895
5891
|
* Parse fields string into a structured object
|
|
@@ -9925,7 +8921,7 @@ const RedditAds = (function() {
|
|
|
9925
8921
|
* @param {Date|null} startDate - The start date for data fetching.
|
|
9926
8922
|
* @returns {Array} An array of data records.
|
|
9927
8923
|
*/
|
|
9928
|
-
fetchData(nodeName, accountId, fields, startDate = null) {
|
|
8924
|
+
async fetchData(nodeName, accountId, fields, startDate = null) {
|
|
9929
8925
|
var _a;
|
|
9930
8926
|
console.log(`Fetching data from ${nodeName}/${accountId} for ${startDate}`);
|
|
9931
8927
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -9933,7 +8929,7 @@ const RedditAds = (function() {
|
|
|
9933
8929
|
if (missingKeys.length > 0) {
|
|
9934
8930
|
throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
|
|
9935
8931
|
}
|
|
9936
|
-
const tokenResponse = this.getRedditAccessToken(
|
|
8932
|
+
const tokenResponse = await this.getRedditAccessToken(
|
|
9937
8933
|
this.config.ClientId.value,
|
|
9938
8934
|
this.config.ClientSecret.value,
|
|
9939
8935
|
this.config.RedirectUri.value,
|
|
@@ -9943,7 +8939,7 @@ const RedditAds = (function() {
|
|
|
9943
8939
|
this.config.AccessToken.value = tokenResponse.accessToken;
|
|
9944
8940
|
}
|
|
9945
8941
|
const baseUrl = "https://ads-api.reddit.com/api/v3/";
|
|
9946
|
-
let formattedDate = startDate ?
|
|
8942
|
+
let formattedDate = startDate ? DateUtils3.formatDate(startDate) : null;
|
|
9947
8943
|
let headers = {
|
|
9948
8944
|
"Accept": "application/json",
|
|
9949
8945
|
"User-Agent": this.config.UserAgent.value,
|
|
@@ -9969,8 +8965,9 @@ const RedditAds = (function() {
|
|
|
9969
8965
|
let nextPageURL = finalUrl;
|
|
9970
8966
|
while (nextPageURL) {
|
|
9971
8967
|
try {
|
|
9972
|
-
const response = this.urlFetchWithRetry(nextPageURL, options);
|
|
9973
|
-
const
|
|
8968
|
+
const response = await this.urlFetchWithRetry(nextPageURL, options);
|
|
8969
|
+
const text = await response.getContentText();
|
|
8970
|
+
const jsonData = JSON.parse(text);
|
|
9974
8971
|
if ("data" in jsonData) {
|
|
9975
8972
|
nextPageURL = jsonData.pagination ? jsonData.pagination.next_url : null;
|
|
9976
8973
|
if (jsonData && jsonData.data && jsonData.data.metrics) {
|
|
@@ -9987,7 +8984,7 @@ const RedditAds = (function() {
|
|
|
9987
8984
|
}
|
|
9988
8985
|
} catch (error) {
|
|
9989
8986
|
if (error.statusCode === HTTP_STATUS2.UNAUTHORIZED) {
|
|
9990
|
-
const newTokenResponse = this.getRedditAccessToken(
|
|
8987
|
+
const newTokenResponse = await this.getRedditAccessToken(
|
|
9991
8988
|
this.config.ClientId.value,
|
|
9992
8989
|
this.config.ClientSecret.value,
|
|
9993
8990
|
this.config.RedirectUri.value,
|
|
@@ -10014,12 +9011,12 @@ const RedditAds = (function() {
|
|
|
10014
9011
|
* @param {string} refreshToken - The refresh token.
|
|
10015
9012
|
* @returns {Object} An object with a success flag and either the access token or an error message.
|
|
10016
9013
|
*/
|
|
10017
|
-
getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
|
|
9014
|
+
async getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
|
|
10018
9015
|
const url = "https://www.reddit.com/api/v1/access_token";
|
|
10019
9016
|
const headers = {
|
|
10020
9017
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
10021
9018
|
"User-Agent": this.config.UserAgent.value,
|
|
10022
|
-
"Authorization": "Basic " +
|
|
9019
|
+
"Authorization": "Basic " + CryptoUtils3.base64Encode(clientId + ":" + clientSecret)
|
|
10023
9020
|
};
|
|
10024
9021
|
const payload = {
|
|
10025
9022
|
"grant_type": "refresh_token",
|
|
@@ -10035,8 +9032,8 @@ const RedditAds = (function() {
|
|
|
10035
9032
|
muteHttpExceptions: true
|
|
10036
9033
|
};
|
|
10037
9034
|
try {
|
|
10038
|
-
const response =
|
|
10039
|
-
const result = response.getContentText();
|
|
9035
|
+
const response = await HttpUtils3.fetch(url, options);
|
|
9036
|
+
const result = await response.getContentText();
|
|
10040
9037
|
const json = JSON.parse(result);
|
|
10041
9038
|
if (json.error) {
|
|
10042
9039
|
return { success: false, message: json.error };
|
|
@@ -10421,7 +9418,7 @@ const RedditAds = (function() {
|
|
|
10421
9418
|
}
|
|
10422
9419
|
};
|
|
10423
9420
|
var RedditAdsConnector = class RedditAdsConnector extends AbstractConnector3 {
|
|
10424
|
-
constructor(config, source, storageName = "
|
|
9421
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10425
9422
|
super(config, source, null, runConfig);
|
|
10426
9423
|
this.storageName = storageName;
|
|
10427
9424
|
}
|
|
@@ -10429,12 +9426,12 @@ const RedditAds = (function() {
|
|
|
10429
9426
|
* Main method - entry point for the import process
|
|
10430
9427
|
* Processes all nodes defined in the fields configuration
|
|
10431
9428
|
*/
|
|
10432
|
-
startImportProcess() {
|
|
9429
|
+
async startImportProcess() {
|
|
10433
9430
|
const fields = RedditAdsHelper.parseFields(this.config.Fields.value);
|
|
10434
9431
|
const accountIds = RedditAdsHelper.parseAccountIds(this.config.AccountIDs.value);
|
|
10435
9432
|
for (const accountId of accountIds) {
|
|
10436
9433
|
for (const nodeName in fields) {
|
|
10437
|
-
this.processNode({
|
|
9434
|
+
await this.processNode({
|
|
10438
9435
|
nodeName,
|
|
10439
9436
|
accountId,
|
|
10440
9437
|
fields: fields[nodeName] || []
|
|
@@ -10449,15 +9446,15 @@ const RedditAds = (function() {
|
|
|
10449
9446
|
* @param {string} options.accountId - Account ID
|
|
10450
9447
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10451
9448
|
*/
|
|
10452
|
-
processNode({ nodeName, accountId, fields }) {
|
|
9449
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
10453
9450
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
10454
|
-
this.processTimeSeriesNode({
|
|
9451
|
+
await this.processTimeSeriesNode({
|
|
10455
9452
|
nodeName,
|
|
10456
9453
|
accountId,
|
|
10457
9454
|
fields
|
|
10458
9455
|
});
|
|
10459
9456
|
} else {
|
|
10460
|
-
this.processCatalogNode({
|
|
9457
|
+
await this.processCatalogNode({
|
|
10461
9458
|
nodeName,
|
|
10462
9459
|
accountId,
|
|
10463
9460
|
fields
|
|
@@ -10472,7 +9469,7 @@ const RedditAds = (function() {
|
|
|
10472
9469
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10473
9470
|
* @param {Object} options.storage - Storage instance
|
|
10474
9471
|
*/
|
|
10475
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
9472
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
10476
9473
|
var _a;
|
|
10477
9474
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
10478
9475
|
if (daysToFetch <= 0) {
|
|
@@ -10482,13 +9479,14 @@ const RedditAds = (function() {
|
|
|
10482
9479
|
for (let i = 0; i < daysToFetch; i++) {
|
|
10483
9480
|
const currentDate = new Date(startDate);
|
|
10484
9481
|
currentDate.setDate(currentDate.getDate() + i);
|
|
10485
|
-
const formattedDate =
|
|
9482
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
10486
9483
|
this.config.logMessage(`Start importing data for ${formattedDate}: ${accountId}/${nodeName}`);
|
|
10487
|
-
const data = this.source.fetchData(nodeName, accountId, fields, currentDate);
|
|
9484
|
+
const data = await this.source.fetchData(nodeName, accountId, fields, currentDate);
|
|
10488
9485
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
10489
9486
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10490
9487
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10491
|
-
this.getStorageByNode(nodeName)
|
|
9488
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
9489
|
+
await storage.saveData(preparedData);
|
|
10492
9490
|
}
|
|
10493
9491
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
10494
9492
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -10503,13 +9501,14 @@ const RedditAds = (function() {
|
|
|
10503
9501
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10504
9502
|
* @param {Object} options.storage - Storage instance
|
|
10505
9503
|
*/
|
|
10506
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
9504
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
10507
9505
|
var _a;
|
|
10508
|
-
const data = this.source.fetchData(nodeName, accountId, fields);
|
|
9506
|
+
const data = await this.source.fetchData(nodeName, accountId, fields);
|
|
10509
9507
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for account ${accountId}` : `No records have been fetched`);
|
|
10510
9508
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10511
9509
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10512
|
-
this.getStorageByNode(nodeName)
|
|
9510
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
9511
|
+
await storage.saveData(preparedData);
|
|
10513
9512
|
}
|
|
10514
9513
|
}
|
|
10515
9514
|
/**
|
|
@@ -10517,7 +9516,7 @@ const RedditAds = (function() {
|
|
|
10517
9516
|
* @param {string} nodeName - Name of the node
|
|
10518
9517
|
* @returns {Object} Storage instance
|
|
10519
9518
|
*/
|
|
10520
|
-
getStorageByNode(nodeName) {
|
|
9519
|
+
async getStorageByNode(nodeName) {
|
|
10521
9520
|
if (!("storages" in this)) {
|
|
10522
9521
|
this.storages = {};
|
|
10523
9522
|
}
|
|
@@ -10535,6 +9534,7 @@ const RedditAds = (function() {
|
|
|
10535
9534
|
this.source.fieldsSchema[nodeName].fields,
|
|
10536
9535
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
10537
9536
|
);
|
|
9537
|
+
await this.storages[nodeName].init();
|
|
10538
9538
|
}
|
|
10539
9539
|
return this.storages[nodeName];
|
|
10540
9540
|
}
|
|
@@ -10551,7 +9551,7 @@ const RedditAds = (function() {
|
|
|
10551
9551
|
};
|
|
10552
9552
|
})();
|
|
10553
9553
|
const OpenHolidays = (function() {
|
|
10554
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
9554
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
10555
9555
|
var publicHolidaysFields = {
|
|
10556
9556
|
id: {
|
|
10557
9557
|
type: "string",
|
|
@@ -10659,10 +9659,10 @@ const OpenHolidays = (function() {
|
|
|
10659
9659
|
* @param {string} [opts.end_time]
|
|
10660
9660
|
* @returns {Array<Object>}
|
|
10661
9661
|
*/
|
|
10662
|
-
fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
9662
|
+
async fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
10663
9663
|
switch (nodeName) {
|
|
10664
9664
|
case "publicHolidays":
|
|
10665
|
-
return this._fetchPublicHolidays({ fields, start_time, end_time });
|
|
9665
|
+
return await this._fetchPublicHolidays({ fields, start_time, end_time });
|
|
10666
9666
|
default:
|
|
10667
9667
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
10668
9668
|
}
|
|
@@ -10675,10 +9675,10 @@ const OpenHolidays = (function() {
|
|
|
10675
9675
|
* @param {string} options.end_time - End date for data fetch (YYYY-MM-DD format)
|
|
10676
9676
|
* @returns {Array} Array of holiday data
|
|
10677
9677
|
*/
|
|
10678
|
-
_fetchPublicHolidays({ fields, start_time, end_time }) {
|
|
9678
|
+
async _fetchPublicHolidays({ fields, start_time, end_time }) {
|
|
10679
9679
|
let countryIsoCode = this.config.countryIsoCode.value;
|
|
10680
9680
|
let languageIsoCode = this.config.languageIsoCode.value;
|
|
10681
|
-
const holidays = this.makeRequest({
|
|
9681
|
+
const holidays = await this.makeRequest({
|
|
10682
9682
|
endpoint: `PublicHolidays?countryIsoCode=${countryIsoCode}&languageIsoCode=${languageIsoCode}&validFrom=${start_time}&validTo=${end_time}`
|
|
10683
9683
|
});
|
|
10684
9684
|
if (!holidays || !holidays.length) {
|
|
@@ -10705,12 +9705,13 @@ const OpenHolidays = (function() {
|
|
|
10705
9705
|
* @param {string} options.endpoint - API endpoint path (e.g., "PublicHolidays?countryIsoCode=CH&...")
|
|
10706
9706
|
* @returns {Object} - API response parsed from JSON
|
|
10707
9707
|
*/
|
|
10708
|
-
makeRequest({ endpoint }) {
|
|
9708
|
+
async makeRequest({ endpoint }) {
|
|
10709
9709
|
const baseUrl = "https://openholidaysapi.org/";
|
|
10710
9710
|
const url = `${baseUrl}${endpoint}`;
|
|
10711
9711
|
console.log(`OpenHolidays API Request URL:`, url);
|
|
10712
|
-
const response =
|
|
10713
|
-
const
|
|
9712
|
+
const response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
9713
|
+
const text = await response.getContentText();
|
|
9714
|
+
const result = JSON.parse(text);
|
|
10714
9715
|
return result;
|
|
10715
9716
|
}
|
|
10716
9717
|
/**
|
|
@@ -10736,7 +9737,7 @@ const OpenHolidays = (function() {
|
|
|
10736
9737
|
}
|
|
10737
9738
|
};
|
|
10738
9739
|
var OpenHolidaysConnector = class OpenHolidaysConnector extends AbstractConnector3 {
|
|
10739
|
-
constructor(config, source, storageName = "
|
|
9740
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10740
9741
|
super(config, source, null, runConfig);
|
|
10741
9742
|
this.storageName = storageName;
|
|
10742
9743
|
}
|
|
@@ -10744,10 +9745,10 @@ const OpenHolidays = (function() {
|
|
|
10744
9745
|
* Main method - entry point for the import process
|
|
10745
9746
|
* Processes all nodes defined in the fields configuration
|
|
10746
9747
|
*/
|
|
10747
|
-
startImportProcess() {
|
|
9748
|
+
async startImportProcess() {
|
|
10748
9749
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
10749
9750
|
for (const nodeName in fields) {
|
|
10750
|
-
this.processNode({
|
|
9751
|
+
await this.processNode({
|
|
10751
9752
|
nodeName,
|
|
10752
9753
|
fields: fields[nodeName] || []
|
|
10753
9754
|
});
|
|
@@ -10759,12 +9760,12 @@ const OpenHolidays = (function() {
|
|
|
10759
9760
|
* @param {string} options.nodeName - Name of the node to process
|
|
10760
9761
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10761
9762
|
*/
|
|
10762
|
-
processNode({ nodeName, fields }) {
|
|
9763
|
+
async processNode({ nodeName, fields }) {
|
|
10763
9764
|
const storage = this.getStorageByNode(nodeName);
|
|
10764
9765
|
if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
|
|
10765
|
-
this.processTimeSeriesNode({ nodeName, fields, storage });
|
|
9766
|
+
await this.processTimeSeriesNode({ nodeName, fields, storage });
|
|
10766
9767
|
} else {
|
|
10767
|
-
this.processCatalogNode({ nodeName, fields, storage });
|
|
9768
|
+
await this.processCatalogNode({ nodeName, fields, storage });
|
|
10768
9769
|
}
|
|
10769
9770
|
}
|
|
10770
9771
|
/**
|
|
@@ -10774,14 +9775,14 @@ const OpenHolidays = (function() {
|
|
|
10774
9775
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10775
9776
|
* @param {Object} options.storage - Storage instance
|
|
10776
9777
|
*/
|
|
10777
|
-
processTimeSeriesNode({ nodeName, fields, storage }) {
|
|
9778
|
+
async processTimeSeriesNode({ nodeName, fields, storage }) {
|
|
10778
9779
|
var _a;
|
|
10779
9780
|
const dateRange = this.prepareDateRange();
|
|
10780
9781
|
if (!dateRange) {
|
|
10781
9782
|
console.log("No date range available for time series data");
|
|
10782
9783
|
return;
|
|
10783
9784
|
}
|
|
10784
|
-
const data = this.source.fetchData({
|
|
9785
|
+
const data = await this.source.fetchData({
|
|
10785
9786
|
nodeName,
|
|
10786
9787
|
start_time: dateRange.startDate,
|
|
10787
9788
|
end_time: dateRange.endDate,
|
|
@@ -10790,7 +9791,7 @@ const OpenHolidays = (function() {
|
|
|
10790
9791
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
|
|
10791
9792
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10792
9793
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10793
|
-
storage.saveData(preparedData);
|
|
9794
|
+
await storage.saveData(preparedData);
|
|
10794
9795
|
}
|
|
10795
9796
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
10796
9797
|
this.config.updateLastRequstedDate(new Date(dateRange.endDate));
|
|
@@ -10803,7 +9804,7 @@ const OpenHolidays = (function() {
|
|
|
10803
9804
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10804
9805
|
* @param {Object} options.storage - Storage instance
|
|
10805
9806
|
*/
|
|
10806
|
-
processCatalogNode({ nodeName, fields, storage }) {
|
|
9807
|
+
async processCatalogNode({ nodeName, fields, storage }) {
|
|
10807
9808
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
10808
9809
|
}
|
|
10809
9810
|
/**
|
|
@@ -10811,7 +9812,7 @@ const OpenHolidays = (function() {
|
|
|
10811
9812
|
* @param {string} nodeName - Name of the node
|
|
10812
9813
|
* @returns {Object} Storage instance
|
|
10813
9814
|
*/
|
|
10814
|
-
getStorageByNode(nodeName) {
|
|
9815
|
+
async getStorageByNode(nodeName) {
|
|
10815
9816
|
if (!("storages" in this)) {
|
|
10816
9817
|
this.storages = {};
|
|
10817
9818
|
}
|
|
@@ -10829,6 +9830,7 @@ const OpenHolidays = (function() {
|
|
|
10829
9830
|
this.source.fieldsSchema[nodeName].fields,
|
|
10830
9831
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
10831
9832
|
);
|
|
9833
|
+
await this.storages[nodeName].init();
|
|
10832
9834
|
}
|
|
10833
9835
|
return this.storages[nodeName];
|
|
10834
9836
|
}
|
|
@@ -10844,8 +9846,8 @@ const OpenHolidays = (function() {
|
|
|
10844
9846
|
const endDate = new Date(startDate);
|
|
10845
9847
|
endDate.setDate(endDate.getDate() + daysToFetch - 1);
|
|
10846
9848
|
return {
|
|
10847
|
-
startDate:
|
|
10848
|
-
endDate:
|
|
9849
|
+
startDate: DateUtils3.formatDate(startDate),
|
|
9850
|
+
endDate: DateUtils3.formatDate(endDate)
|
|
10849
9851
|
};
|
|
10850
9852
|
}
|
|
10851
9853
|
};
|
|
@@ -10861,7 +9863,7 @@ const OpenHolidays = (function() {
|
|
|
10861
9863
|
};
|
|
10862
9864
|
})();
|
|
10863
9865
|
const OpenExchangeRates = (function() {
|
|
10864
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
9866
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
10865
9867
|
var historicalFields = {
|
|
10866
9868
|
date: {
|
|
10867
9869
|
type: "DATE",
|
|
@@ -10949,12 +9951,12 @@ const OpenExchangeRates = (function() {
|
|
|
10949
9951
|
this.fieldsSchema = OpenExchangeRatesFieldsSchema;
|
|
10950
9952
|
}
|
|
10951
9953
|
/*
|
|
10952
|
-
|
|
9954
|
+
@param date The requested date as a Date object
|
|
10953
9955
|
|
|
10954
|
-
|
|
9956
|
+
@return data array
|
|
10955
9957
|
|
|
10956
|
-
|
|
10957
|
-
fetchData(date) {
|
|
9958
|
+
*/
|
|
9959
|
+
async fetchData(date) {
|
|
10958
9960
|
let data = [];
|
|
10959
9961
|
let base = this.config.base.value;
|
|
10960
9962
|
let app_id = this.config.AppId.value;
|
|
@@ -10962,13 +9964,14 @@ const OpenExchangeRates = (function() {
|
|
|
10962
9964
|
if (this.config.Symbols.value) {
|
|
10963
9965
|
symbols = "&symbols=" + String(this.config.Symbols.value).replace(/[^A-Z,]/g, "");
|
|
10964
9966
|
}
|
|
10965
|
-
var date =
|
|
9967
|
+
var date = DateUtils3.formatDate(date);
|
|
10966
9968
|
const urlWithoutKey = `https://openexchangerates.org/api/historical/${date}.json?base=${base}${symbols}`;
|
|
10967
9969
|
console.log(`OpenExchangeRates API URL:`, urlWithoutKey);
|
|
10968
9970
|
const url = `${urlWithoutKey}&app_id=${app_id}`;
|
|
10969
9971
|
this.config.logMessage(`Fetching rates for ${date}`);
|
|
10970
|
-
var response =
|
|
10971
|
-
var
|
|
9972
|
+
var response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
9973
|
+
var text = await response.getContentText();
|
|
9974
|
+
var historical = JSON.parse(text);
|
|
10972
9975
|
for (var currency in historical["rates"]) {
|
|
10973
9976
|
data.push({
|
|
10974
9977
|
date: new Date(date),
|
|
@@ -10981,7 +9984,7 @@ const OpenExchangeRates = (function() {
|
|
|
10981
9984
|
}
|
|
10982
9985
|
};
|
|
10983
9986
|
var OpenExchangeRatesConnector = class OpenExchangeRatesConnector extends AbstractConnector3 {
|
|
10984
|
-
constructor(config, source, storageName = "
|
|
9987
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10985
9988
|
super(config, source, null, runConfig);
|
|
10986
9989
|
this.storageName = storageName;
|
|
10987
9990
|
}
|
|
@@ -10994,11 +9997,11 @@ const OpenExchangeRates = (function() {
|
|
|
10994
9997
|
* Main method - entry point for the import process
|
|
10995
9998
|
* Processes all nodes defined in the fields configuration
|
|
10996
9999
|
*/
|
|
10997
|
-
startImportProcess() {
|
|
10000
|
+
async startImportProcess() {
|
|
10998
10001
|
var _a;
|
|
10999
10002
|
const fields = ConnectorUtils.parseFields((_a = this.config.Fields) == null ? void 0 : _a.value);
|
|
11000
10003
|
for (const nodeName in fields) {
|
|
11001
|
-
this.processNode({
|
|
10004
|
+
await this.processNode({
|
|
11002
10005
|
nodeName,
|
|
11003
10006
|
fields: fields[nodeName] || []
|
|
11004
10007
|
});
|
|
@@ -11010,11 +10013,11 @@ const OpenExchangeRates = (function() {
|
|
|
11010
10013
|
* @param {string} options.nodeName - Name of the node to process
|
|
11011
10014
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11012
10015
|
*/
|
|
11013
|
-
processNode({ nodeName, fields }) {
|
|
10016
|
+
async processNode({ nodeName, fields }) {
|
|
11014
10017
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
11015
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
10018
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
11016
10019
|
} else {
|
|
11017
|
-
this.processCatalogNode({ nodeName, fields });
|
|
10020
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
11018
10021
|
}
|
|
11019
10022
|
}
|
|
11020
10023
|
/**
|
|
@@ -11023,7 +10026,7 @@ const OpenExchangeRates = (function() {
|
|
|
11023
10026
|
* @param {string} options.nodeName - Name of the node
|
|
11024
10027
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11025
10028
|
*/
|
|
11026
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
10029
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
11027
10030
|
var _a;
|
|
11028
10031
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
11029
10032
|
if (daysToFetch <= 0) {
|
|
@@ -11033,11 +10036,12 @@ const OpenExchangeRates = (function() {
|
|
|
11033
10036
|
for (let daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
11034
10037
|
const currentDate = new Date(startDate);
|
|
11035
10038
|
currentDate.setDate(currentDate.getDate() + daysShift);
|
|
11036
|
-
let data = this.source.fetchData(currentDate);
|
|
10039
|
+
let data = await this.source.fetchData(currentDate);
|
|
11037
10040
|
this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
|
|
11038
10041
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
11039
10042
|
const preparedData = data.length ? data : [];
|
|
11040
|
-
this.getStorageByNode(nodeName)
|
|
10043
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
10044
|
+
await storage.saveData(preparedData);
|
|
11041
10045
|
}
|
|
11042
10046
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
11043
10047
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -11050,7 +10054,7 @@ const OpenExchangeRates = (function() {
|
|
|
11050
10054
|
* @param {string} options.nodeName - Name of the node
|
|
11051
10055
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11052
10056
|
*/
|
|
11053
|
-
processCatalogNode({ nodeName, fields }) {
|
|
10057
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
11054
10058
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
11055
10059
|
}
|
|
11056
10060
|
//---- getStorageName -------------------------------------------------
|
|
@@ -11062,7 +10066,7 @@ const OpenExchangeRates = (function() {
|
|
|
11062
10066
|
* @return AbstractStorage
|
|
11063
10067
|
*
|
|
11064
10068
|
*/
|
|
11065
|
-
getStorageByNode(nodeName) {
|
|
10069
|
+
async getStorageByNode(nodeName) {
|
|
11066
10070
|
if (!("storages" in this)) {
|
|
11067
10071
|
this.storages = {};
|
|
11068
10072
|
}
|
|
@@ -11080,6 +10084,7 @@ const OpenExchangeRates = (function() {
|
|
|
11080
10084
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
11081
10085
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
11082
10086
|
);
|
|
10087
|
+
await this.storages[nodeName].init();
|
|
11083
10088
|
}
|
|
11084
10089
|
return this.storages[nodeName];
|
|
11085
10090
|
}
|
|
@@ -11096,7 +10101,7 @@ const OpenExchangeRates = (function() {
|
|
|
11096
10101
|
};
|
|
11097
10102
|
})();
|
|
11098
10103
|
const MicrosoftAds = (function() {
|
|
11099
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
10104
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
11100
10105
|
const MicrosoftAdsHelper = {
|
|
11101
10106
|
/**
|
|
11102
10107
|
* Parse fields string into a structured object
|
|
@@ -11119,7 +10124,7 @@ const MicrosoftAds = (function() {
|
|
|
11119
10124
|
* @param {number} [opts.interval=5000]
|
|
11120
10125
|
* @returns {Object}
|
|
11121
10126
|
*/
|
|
11122
|
-
pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
|
|
10127
|
+
async pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
|
|
11123
10128
|
const startTime = Date.now();
|
|
11124
10129
|
const timeout = 15 * 60 * 1e3;
|
|
11125
10130
|
let statusResult;
|
|
@@ -11128,9 +10133,10 @@ const MicrosoftAds = (function() {
|
|
|
11128
10133
|
if (Date.now() - startTime > timeout) {
|
|
11129
10134
|
throw new Error("Polling timed out after 15 minutes");
|
|
11130
10135
|
}
|
|
11131
|
-
|
|
11132
|
-
const response =
|
|
11133
|
-
|
|
10136
|
+
await AsyncUtils3.delay(interval);
|
|
10137
|
+
const response = await HttpUtils3.fetch(url, options);
|
|
10138
|
+
const text = await response.getContentText();
|
|
10139
|
+
statusResult = JSON.parse(text);
|
|
11134
10140
|
} while (!isDone(statusResult));
|
|
11135
10141
|
return statusResult;
|
|
11136
10142
|
} catch (error) {
|
|
@@ -11146,13 +10152,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
11146
10152
|
* @param {string} url
|
|
11147
10153
|
* @returns {Array<Array<string>>}
|
|
11148
10154
|
*/
|
|
11149
|
-
downloadCsvRows(url) {
|
|
11150
|
-
const response =
|
|
11151
|
-
const
|
|
10155
|
+
async downloadCsvRows(url) {
|
|
10156
|
+
const response = await HttpUtils3.fetch(url);
|
|
10157
|
+
const blob = await response.getBlob();
|
|
10158
|
+
const files = FileUtils3.unzip(blob);
|
|
11152
10159
|
const allRows = [];
|
|
11153
10160
|
files.forEach((file) => {
|
|
11154
10161
|
const csvText = file.getDataAsString();
|
|
11155
|
-
const rows =
|
|
10162
|
+
const rows = FileUtils3.parseCsv(csvText);
|
|
11156
10163
|
allRows.push(...rows);
|
|
11157
10164
|
});
|
|
11158
10165
|
return allRows;
|
|
@@ -12279,7 +11286,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12279
11286
|
/**
|
|
12280
11287
|
* Retrieve and store an OAuth access token using the refresh token
|
|
12281
11288
|
*/
|
|
12282
|
-
getAccessToken() {
|
|
11289
|
+
async getAccessToken() {
|
|
12283
11290
|
var _a, _b;
|
|
12284
11291
|
const tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
|
12285
11292
|
const scopes = [
|
|
@@ -12305,8 +11312,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12305
11312
|
body: Object.entries(form).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
|
|
12306
11313
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
12307
11314
|
};
|
|
12308
|
-
const resp =
|
|
12309
|
-
const
|
|
11315
|
+
const resp = await HttpUtils3.fetch(tokenUrl, options);
|
|
11316
|
+
const text = await resp.getContentText();
|
|
11317
|
+
const json = JSON.parse(text);
|
|
12310
11318
|
if (json.error) {
|
|
12311
11319
|
throw new Error(`Token error: ${json.error} - ${json.error_description}`);
|
|
12312
11320
|
}
|
|
@@ -12332,7 +11340,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12332
11340
|
* @param {Function} [opts.onBatchReady] - Optional callback for batch processing
|
|
12333
11341
|
* @returns {Array<Object>}
|
|
12334
11342
|
*/
|
|
12335
|
-
fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
|
|
11343
|
+
async fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
|
|
12336
11344
|
const schema = this.fieldsSchema[nodeName];
|
|
12337
11345
|
if (schema.uniqueKeys) {
|
|
12338
11346
|
const missingKeys = schema.uniqueKeys.filter((key) => !fields.includes(key));
|
|
@@ -12342,12 +11350,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12342
11350
|
}
|
|
12343
11351
|
switch (nodeName) {
|
|
12344
11352
|
case "campaigns":
|
|
12345
|
-
this._fetchCampaignData({ accountId, fields, onBatchReady });
|
|
11353
|
+
await this._fetchCampaignData({ accountId, fields, onBatchReady });
|
|
12346
11354
|
return [];
|
|
12347
11355
|
case "ad_performance_report":
|
|
12348
|
-
return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
11356
|
+
return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
12349
11357
|
case "user_location_performance_report":
|
|
12350
|
-
return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
11358
|
+
return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
12351
11359
|
default:
|
|
12352
11360
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
12353
11361
|
}
|
|
@@ -12361,14 +11369,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12361
11369
|
* @returns {void}
|
|
12362
11370
|
* @private
|
|
12363
11371
|
*/
|
|
12364
|
-
_fetchCampaignData({ accountId, fields, onBatchReady }) {
|
|
12365
|
-
this.getAccessToken();
|
|
11372
|
+
async _fetchCampaignData({ accountId, fields, onBatchReady }) {
|
|
11373
|
+
await this.getAccessToken();
|
|
12366
11374
|
this.config.logMessage(`Fetching Campaigns, AssetGroups and AdGroups for account ${accountId}...`);
|
|
12367
11375
|
const entityTypes = ["Campaigns", "AssetGroups", "AdGroups"];
|
|
12368
11376
|
const allRecords = [];
|
|
12369
11377
|
let campaignRecords = [];
|
|
12370
11378
|
for (const entityType of entityTypes) {
|
|
12371
|
-
const records = this._downloadEntity({
|
|
11379
|
+
const records = await this._downloadEntity({
|
|
12372
11380
|
submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByAccountIds",
|
|
12373
11381
|
submitOpts: {
|
|
12374
11382
|
method: "post",
|
|
@@ -12406,21 +11414,21 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12406
11414
|
}
|
|
12407
11415
|
const filteredMainData = MicrosoftAdsHelper.filterByFields(allRecords, fields);
|
|
12408
11416
|
if (filteredMainData.length > 0) {
|
|
12409
|
-
onBatchReady(filteredMainData);
|
|
11417
|
+
await onBatchReady(filteredMainData);
|
|
12410
11418
|
}
|
|
12411
11419
|
this.config.logMessage(`Fetching Keywords for account ${accountId} (processing by campaigns to avoid size limits)...`);
|
|
12412
11420
|
const campaignIds = MicrosoftAdsHelper.extractCampaignIds(campaignRecords);
|
|
12413
11421
|
this.config.logMessage(`Found ${campaignIds.length} campaigns, fetching Keywords in batches`);
|
|
12414
11422
|
this.config.logMessage(`Campaign IDs: ${campaignIds.slice(0, 10).join(", ")}${campaignIds.length > 10 ? "..." : ""}`);
|
|
12415
11423
|
let totalFetched = 0;
|
|
12416
|
-
this._fetchEntityByCampaigns({
|
|
11424
|
+
await this._fetchEntityByCampaigns({
|
|
12417
11425
|
accountId,
|
|
12418
11426
|
entityType: "Keywords",
|
|
12419
11427
|
campaignIds,
|
|
12420
|
-
onBatchReady: (batchRecords) => {
|
|
11428
|
+
onBatchReady: async (batchRecords) => {
|
|
12421
11429
|
totalFetched += batchRecords.length;
|
|
12422
11430
|
const filteredBatch = MicrosoftAdsHelper.filterByFields(batchRecords, fields);
|
|
12423
|
-
onBatchReady(filteredBatch);
|
|
11431
|
+
await onBatchReady(filteredBatch);
|
|
12424
11432
|
}
|
|
12425
11433
|
});
|
|
12426
11434
|
this.config.logMessage(`${totalFetched} rows of Keywords were fetched for account ${accountId}`);
|
|
@@ -12434,15 +11442,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12434
11442
|
* @returns {Array<Object>}
|
|
12435
11443
|
* @private
|
|
12436
11444
|
*/
|
|
12437
|
-
_downloadEntity({ submitUrl, submitOpts }) {
|
|
12438
|
-
const submitResp =
|
|
12439
|
-
const
|
|
11445
|
+
async _downloadEntity({ submitUrl, submitOpts }) {
|
|
11446
|
+
const submitResp = await HttpUtils3.fetch(submitUrl, submitOpts);
|
|
11447
|
+
const text = await submitResp.getContentText();
|
|
11448
|
+
const requestId = JSON.parse(text).DownloadRequestId;
|
|
12440
11449
|
const pollUrl = "https://bulk.api.bingads.microsoft.com/Bulk/v13/BulkDownloadStatus/Query";
|
|
12441
11450
|
const pollOpts = Object.assign({}, submitOpts, {
|
|
12442
11451
|
payload: JSON.stringify({ RequestId: requestId }),
|
|
12443
11452
|
body: JSON.stringify({ RequestId: requestId })
|
|
12444
11453
|
});
|
|
12445
|
-
const pollResult = MicrosoftAdsHelper.pollUntilStatus({
|
|
11454
|
+
const pollResult = await MicrosoftAdsHelper.pollUntilStatus({
|
|
12446
11455
|
url: pollUrl,
|
|
12447
11456
|
options: pollOpts,
|
|
12448
11457
|
isDone: (status) => {
|
|
@@ -12452,7 +11461,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12452
11461
|
return status.RequestStatus === "Completed";
|
|
12453
11462
|
}
|
|
12454
11463
|
});
|
|
12455
|
-
const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
|
|
11464
|
+
const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
|
|
12456
11465
|
const result = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
|
|
12457
11466
|
return result;
|
|
12458
11467
|
}
|
|
@@ -12466,7 +11475,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12466
11475
|
* @returns {Array<Object>} - Returns empty array if onBatchReady callback is provided, otherwise returns all records
|
|
12467
11476
|
* @private
|
|
12468
11477
|
*/
|
|
12469
|
-
_fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
|
|
11478
|
+
async _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
|
|
12470
11479
|
if (campaignIds.length === 0) {
|
|
12471
11480
|
this.config.logMessage(`No active campaigns found for account ${accountId}, skipping ${entityType} fetch`);
|
|
12472
11481
|
return [];
|
|
@@ -12476,9 +11485,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12476
11485
|
const campaignBatch = campaignIds.slice(i, i + batchSize);
|
|
12477
11486
|
this.config.logMessage(`Fetching ${entityType} for campaigns batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(campaignIds.length / batchSize)} (${campaignBatch.length} campaigns)`);
|
|
12478
11487
|
try {
|
|
12479
|
-
const batchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch });
|
|
11488
|
+
const batchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch });
|
|
12480
11489
|
this.config.logMessage(`Fetched ${batchRecords.length} ${entityType.toLowerCase()} from current batch`);
|
|
12481
|
-
onBatchReady(batchRecords);
|
|
11490
|
+
await onBatchReady(batchRecords);
|
|
12482
11491
|
} catch (error) {
|
|
12483
11492
|
if (error.message && error.message.includes("100MB")) {
|
|
12484
11493
|
const newBatchSize = Math.max(1, Math.floor(batchSize / 2));
|
|
@@ -12486,9 +11495,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12486
11495
|
for (let j = i; j < Math.min(i + batchSize, campaignIds.length); j += newBatchSize) {
|
|
12487
11496
|
const smallerBatch = campaignIds.slice(j, j + newBatchSize);
|
|
12488
11497
|
try {
|
|
12489
|
-
const smallerBatchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
|
|
11498
|
+
const smallerBatchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
|
|
12490
11499
|
this.config.logMessage(`Fetched ${smallerBatchRecords.length} ${entityType.toLowerCase()} from smaller batch (${smallerBatch.length} campaigns)`);
|
|
12491
|
-
onBatchReady(smallerBatchRecords);
|
|
11500
|
+
await onBatchReady(smallerBatchRecords);
|
|
12492
11501
|
} catch (smallerError) {
|
|
12493
11502
|
if (smallerError.message && smallerError.message.includes("100MB")) {
|
|
12494
11503
|
throw new Error(`Failed to fetch ${entityType}: batch size of ${smallerBatch.length} campaigns still exceeds 100MB limit`);
|
|
@@ -12515,7 +11524,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12515
11524
|
* @returns {Array<Object>}
|
|
12516
11525
|
* @private
|
|
12517
11526
|
*/
|
|
12518
|
-
_downloadEntityBatch({ accountId, entityType, campaignBatch }) {
|
|
11527
|
+
async _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
|
|
12519
11528
|
const downloadBody = {
|
|
12520
11529
|
Campaigns: campaignBatch.map((id) => ({
|
|
12521
11530
|
CampaignId: Number(id),
|
|
@@ -12540,7 +11549,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12540
11549
|
payload: JSON.stringify(downloadBody),
|
|
12541
11550
|
body: JSON.stringify(downloadBody)
|
|
12542
11551
|
};
|
|
12543
|
-
return this._downloadEntity({
|
|
11552
|
+
return await this._downloadEntity({
|
|
12544
11553
|
submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByCampaignIds",
|
|
12545
11554
|
submitOpts
|
|
12546
11555
|
});
|
|
@@ -12556,22 +11565,22 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12556
11565
|
* @returns {Array<Object>}
|
|
12557
11566
|
* @private
|
|
12558
11567
|
*/
|
|
12559
|
-
_fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
|
|
12560
|
-
this.getAccessToken();
|
|
11568
|
+
async _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
|
|
11569
|
+
await this.getAccessToken();
|
|
12561
11570
|
const schema = this.fieldsSchema[nodeName];
|
|
12562
|
-
const submitResponse = this._submitReportRequest({
|
|
11571
|
+
const submitResponse = await this._submitReportRequest({
|
|
12563
11572
|
accountId,
|
|
12564
11573
|
fields,
|
|
12565
11574
|
start_time,
|
|
12566
11575
|
end_time,
|
|
12567
11576
|
schema
|
|
12568
11577
|
});
|
|
12569
|
-
const pollResult = this._pollReportStatus({ submitResponse });
|
|
11578
|
+
const pollResult = await this._pollReportStatus({ submitResponse });
|
|
12570
11579
|
if (!pollResult.ReportRequestStatus.ReportDownloadUrl) {
|
|
12571
11580
|
this.config.logMessage(`No data available for the specified time period (${start_time} to ${end_time}). Report status: ${JSON.stringify(pollResult.ReportRequestStatus)}`);
|
|
12572
11581
|
return [];
|
|
12573
11582
|
}
|
|
12574
|
-
const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
|
|
11583
|
+
const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
|
|
12575
11584
|
const records = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
|
|
12576
11585
|
return MicrosoftAdsHelper.filterByFields(records, fields);
|
|
12577
11586
|
}
|
|
@@ -12586,7 +11595,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12586
11595
|
* @returns {Object} - Submit response
|
|
12587
11596
|
* @private
|
|
12588
11597
|
*/
|
|
12589
|
-
_submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
|
|
11598
|
+
async _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
|
|
12590
11599
|
const dateRange = {
|
|
12591
11600
|
CustomDateRangeStart: { Day: new Date(start_time).getDate(), Month: new Date(start_time).getMonth() + 1, Year: new Date(start_time).getFullYear() },
|
|
12592
11601
|
CustomDateRangeEnd: { Day: new Date(end_time).getDate(), Month: new Date(end_time).getMonth() + 1, Year: new Date(end_time).getFullYear() },
|
|
@@ -12619,8 +11628,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12619
11628
|
body: JSON.stringify({ ReportRequest: requestBody })
|
|
12620
11629
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
12621
11630
|
};
|
|
12622
|
-
const submitResp =
|
|
12623
|
-
const submitResponseText = submitResp.getContentText();
|
|
11631
|
+
const submitResp = await HttpUtils3.fetch(submitUrl, submitOpts);
|
|
11632
|
+
const submitResponseText = await submitResp.getContentText();
|
|
12624
11633
|
try {
|
|
12625
11634
|
const submitResponse = JSON.parse(submitResponseText);
|
|
12626
11635
|
if (submitResponse.OperationErrors && submitResponse.OperationErrors.length > 0) {
|
|
@@ -12644,7 +11653,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12644
11653
|
* @returns {Object} - Poll result with report status
|
|
12645
11654
|
* @private
|
|
12646
11655
|
*/
|
|
12647
|
-
_pollReportStatus({ submitResponse }) {
|
|
11656
|
+
async _pollReportStatus({ submitResponse }) {
|
|
12648
11657
|
const pollUrl = "https://reporting.api.bingads.microsoft.com/Reporting/v13/GenerateReport/Poll";
|
|
12649
11658
|
const submitResponseText = JSON.stringify(submitResponse);
|
|
12650
11659
|
const pollOpts = {
|
|
@@ -12660,7 +11669,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12660
11669
|
payload: submitResponseText,
|
|
12661
11670
|
body: submitResponseText
|
|
12662
11671
|
};
|
|
12663
|
-
return MicrosoftAdsHelper.pollUntilStatus({
|
|
11672
|
+
return await MicrosoftAdsHelper.pollUntilStatus({
|
|
12664
11673
|
url: pollUrl,
|
|
12665
11674
|
options: pollOpts,
|
|
12666
11675
|
isDone: (status) => {
|
|
@@ -12673,7 +11682,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12673
11682
|
}
|
|
12674
11683
|
};
|
|
12675
11684
|
var MicrosoftAdsConnector = class MicrosoftAdsConnector extends AbstractConnector3 {
|
|
12676
|
-
constructor(config, source, storageName = "
|
|
11685
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
12677
11686
|
super(config, source, null, runConfig);
|
|
12678
11687
|
this.storageName = storageName;
|
|
12679
11688
|
}
|
|
@@ -12681,10 +11690,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12681
11690
|
* Main method - entry point for the import process
|
|
12682
11691
|
* Processes all nodes defined in the fields configuration
|
|
12683
11692
|
*/
|
|
12684
|
-
startImportProcess() {
|
|
11693
|
+
async startImportProcess() {
|
|
12685
11694
|
const fields = MicrosoftAdsHelper.parseFields(this.config.Fields.value);
|
|
12686
11695
|
for (const nodeName in fields) {
|
|
12687
|
-
this.processNode({
|
|
11696
|
+
await this.processNode({
|
|
12688
11697
|
nodeName,
|
|
12689
11698
|
accountId: this.config.AccountID.value,
|
|
12690
11699
|
fields: fields[nodeName] || []
|
|
@@ -12698,15 +11707,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12698
11707
|
* @param {string} options.accountId - Account ID
|
|
12699
11708
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12700
11709
|
*/
|
|
12701
|
-
processNode({ nodeName, accountId, fields }) {
|
|
11710
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
12702
11711
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
12703
|
-
this.processTimeSeriesNode({
|
|
11712
|
+
await this.processTimeSeriesNode({
|
|
12704
11713
|
nodeName,
|
|
12705
11714
|
accountId,
|
|
12706
11715
|
fields
|
|
12707
11716
|
});
|
|
12708
11717
|
} else {
|
|
12709
|
-
this.processCatalogNode({
|
|
11718
|
+
await this.processCatalogNode({
|
|
12710
11719
|
nodeName,
|
|
12711
11720
|
accountId,
|
|
12712
11721
|
fields
|
|
@@ -12721,7 +11730,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12721
11730
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12722
11731
|
* @param {Object} options.storage - Storage instance
|
|
12723
11732
|
*/
|
|
12724
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
11733
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
12725
11734
|
var _a;
|
|
12726
11735
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
12727
11736
|
if (daysToFetch <= 0) {
|
|
@@ -12731,9 +11740,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12731
11740
|
for (let dayOffset = 0; dayOffset < daysToFetch; dayOffset++) {
|
|
12732
11741
|
const currentDate = new Date(startDate);
|
|
12733
11742
|
currentDate.setDate(currentDate.getDate() + dayOffset);
|
|
12734
|
-
const formattedDate =
|
|
11743
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
12735
11744
|
this.config.logMessage(`Processing ${nodeName} for ${accountId} on ${formattedDate} (day ${dayOffset + 1} of ${daysToFetch})`);
|
|
12736
|
-
const data = this.source.fetchData({
|
|
11745
|
+
const data = await this.source.fetchData({
|
|
12737
11746
|
nodeName,
|
|
12738
11747
|
accountId,
|
|
12739
11748
|
start_time: formattedDate,
|
|
@@ -12743,7 +11752,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12743
11752
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
|
|
12744
11753
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
12745
11754
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
12746
|
-
this.getStorageByNode(nodeName)
|
|
11755
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11756
|
+
await storage.saveData(preparedData);
|
|
12747
11757
|
data.length && this.config.logMessage(`Successfully saved ${data.length} rows for ${formattedDate}`);
|
|
12748
11758
|
}
|
|
12749
11759
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
@@ -12759,22 +11769,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12759
11769
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12760
11770
|
* @param {Object} options.storage - Storage instance
|
|
12761
11771
|
*/
|
|
12762
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
11772
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
12763
11773
|
var _a;
|
|
12764
|
-
const data = this.source.fetchData({
|
|
11774
|
+
const data = await this.source.fetchData({
|
|
12765
11775
|
nodeName,
|
|
12766
11776
|
accountId,
|
|
12767
11777
|
fields,
|
|
12768
|
-
onBatchReady: (batchData) => {
|
|
11778
|
+
onBatchReady: async (batchData) => {
|
|
12769
11779
|
this.config.logMessage(`Saving batch of ${batchData.length} records to storage`);
|
|
12770
11780
|
const preparedData = this.addMissingFieldsToData(batchData, fields);
|
|
12771
|
-
this.getStorageByNode(nodeName)
|
|
11781
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11782
|
+
await storage.saveData(preparedData);
|
|
12772
11783
|
}
|
|
12773
11784
|
});
|
|
12774
11785
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
|
|
12775
11786
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
12776
11787
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
12777
|
-
this.getStorageByNode(nodeName)
|
|
11788
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11789
|
+
await storage.saveData(preparedData);
|
|
12778
11790
|
}
|
|
12779
11791
|
}
|
|
12780
11792
|
/**
|
|
@@ -12782,7 +11794,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12782
11794
|
* @param {string} nodeName - Name of the node
|
|
12783
11795
|
* @returns {Object} Storage instance
|
|
12784
11796
|
*/
|
|
12785
|
-
getStorageByNode(nodeName) {
|
|
11797
|
+
async getStorageByNode(nodeName) {
|
|
12786
11798
|
if (!("storages" in this)) {
|
|
12787
11799
|
this.storages = {};
|
|
12788
11800
|
}
|
|
@@ -12800,6 +11812,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12800
11812
|
this.source.fieldsSchema[nodeName].fields,
|
|
12801
11813
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
12802
11814
|
);
|
|
11815
|
+
await this.storages[nodeName].init();
|
|
12803
11816
|
}
|
|
12804
11817
|
return this.storages[nodeName];
|
|
12805
11818
|
}
|
|
@@ -12816,7 +11829,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12816
11829
|
};
|
|
12817
11830
|
})();
|
|
12818
11831
|
const LinkedInPages = (function() {
|
|
12819
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
11832
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
12820
11833
|
var followerStatisticsTimeBoundFields = {
|
|
12821
11834
|
"organization_urn": {
|
|
12822
11835
|
"description": "Organization URN",
|
|
@@ -12998,7 +12011,7 @@ const LinkedInPages = (function() {
|
|
|
12998
12011
|
* @param {Object} params - Additional parameters for the request
|
|
12999
12012
|
* @returns {Array} - Array of processed data objects
|
|
13000
12013
|
*/
|
|
13001
|
-
fetchData(nodeName, urn, params = {}) {
|
|
12014
|
+
async fetchData(nodeName, urn, params = {}) {
|
|
13002
12015
|
var _a;
|
|
13003
12016
|
const fields = params.fields || [];
|
|
13004
12017
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -13008,7 +12021,7 @@ const LinkedInPages = (function() {
|
|
|
13008
12021
|
}
|
|
13009
12022
|
switch (nodeName) {
|
|
13010
12023
|
case "follower_statistics_time_bound":
|
|
13011
|
-
return this.fetchOrganizationStats({
|
|
12024
|
+
return await this.fetchOrganizationStats({
|
|
13012
12025
|
urn,
|
|
13013
12026
|
nodeName,
|
|
13014
12027
|
endpoint: "organizationalEntityFollowerStatistics",
|
|
@@ -13017,7 +12030,7 @@ const LinkedInPages = (function() {
|
|
|
13017
12030
|
params
|
|
13018
12031
|
});
|
|
13019
12032
|
case "follower_statistics":
|
|
13020
|
-
return this.fetchOrganizationStats({
|
|
12033
|
+
return await this.fetchOrganizationStats({
|
|
13021
12034
|
urn,
|
|
13022
12035
|
nodeName,
|
|
13023
12036
|
endpoint: "organizationalEntityFollowerStatistics",
|
|
@@ -13042,7 +12055,7 @@ const LinkedInPages = (function() {
|
|
|
13042
12055
|
* @param {Array} [options.params.fields] - Additional parameters including fields
|
|
13043
12056
|
* @returns {Array} - Processed statistics data
|
|
13044
12057
|
*/
|
|
13045
|
-
fetchOrganizationStats(options) {
|
|
12058
|
+
async fetchOrganizationStats(options) {
|
|
13046
12059
|
const { urn, nodeName, endpoint, entityParam, formatter, params } = options;
|
|
13047
12060
|
const orgUrn = `urn:li:organization:${urn}`;
|
|
13048
12061
|
const encodedUrn = encodeURIComponent(orgUrn);
|
|
@@ -13053,7 +12066,7 @@ const LinkedInPages = (function() {
|
|
|
13053
12066
|
const endTimestamp = new Date(params.endDate).getTime();
|
|
13054
12067
|
url += `&timeIntervals=(timeRange:(start:${startTimestamp},end:${endTimestamp}),timeGranularityType:DAY)`;
|
|
13055
12068
|
}
|
|
13056
|
-
const response = this.makeRequest(url);
|
|
12069
|
+
const response = await this.makeRequest(url);
|
|
13057
12070
|
const elements = response.elements || [];
|
|
13058
12071
|
if (elements.length === 0) {
|
|
13059
12072
|
return [];
|
|
@@ -13070,9 +12083,9 @@ const LinkedInPages = (function() {
|
|
|
13070
12083
|
* @param {Object} headers - Optional additional headers
|
|
13071
12084
|
* @returns {Object} - API response parsed from JSON
|
|
13072
12085
|
*/
|
|
13073
|
-
makeRequest(url) {
|
|
12086
|
+
async makeRequest(url) {
|
|
13074
12087
|
console.log(`LinkedIn Pages API URL:`, url);
|
|
13075
|
-
OAuthUtils.getAccessToken({
|
|
12088
|
+
await OAuthUtils.getAccessToken({
|
|
13076
12089
|
config: this.config,
|
|
13077
12090
|
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
|
|
13078
12091
|
formData: {
|
|
@@ -13087,12 +12100,13 @@ const LinkedInPages = (function() {
|
|
|
13087
12100
|
"X-RestLi-Protocol-Version": "2.0.0"
|
|
13088
12101
|
};
|
|
13089
12102
|
const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
|
|
13090
|
-
const response =
|
|
13091
|
-
const result =
|
|
13092
|
-
|
|
13093
|
-
|
|
12103
|
+
const response = await HttpUtils3.fetch(authUrl, { headers });
|
|
12104
|
+
const result = await response.getContentText();
|
|
12105
|
+
const parsedResult = JSON.parse(result);
|
|
12106
|
+
if (parsedResult.status && parsedResult.status >= HTTP_STATUS2.BAD_REQUEST) {
|
|
12107
|
+
throw new Error(`LinkedIn API Error: ${parsedResult.message || "Unknown error"} (Status: ${parsedResult.status})`);
|
|
13094
12108
|
}
|
|
13095
|
-
return
|
|
12109
|
+
return parsedResult;
|
|
13096
12110
|
}
|
|
13097
12111
|
/**
|
|
13098
12112
|
* Process time-bound statistics data
|
|
@@ -13186,7 +12200,7 @@ const LinkedInPages = (function() {
|
|
|
13186
12200
|
}
|
|
13187
12201
|
};
|
|
13188
12202
|
var LinkedInPagesConnector = class LinkedInPagesConnector extends AbstractConnector3 {
|
|
13189
|
-
constructor(config, source, storageName = "
|
|
12203
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
13190
12204
|
super(config, source, null, runConfig);
|
|
13191
12205
|
this.storageName = storageName;
|
|
13192
12206
|
}
|
|
@@ -13194,11 +12208,11 @@ const LinkedInPages = (function() {
|
|
|
13194
12208
|
* Main method - entry point for the import process
|
|
13195
12209
|
* Processes all nodes defined in the fields configuration
|
|
13196
12210
|
*/
|
|
13197
|
-
startImportProcess() {
|
|
12211
|
+
async startImportProcess() {
|
|
13198
12212
|
const urns = FormatUtils.parseIds(this.config.OrganizationURNs.value, { prefix: "urn:li:organization:" });
|
|
13199
12213
|
const dataSources = FormatUtils.parseFields(this.config.Fields.value);
|
|
13200
12214
|
for (const nodeName in dataSources) {
|
|
13201
|
-
this.processNode({
|
|
12215
|
+
await this.processNode({
|
|
13202
12216
|
nodeName,
|
|
13203
12217
|
urns,
|
|
13204
12218
|
fields: dataSources[nodeName] || []
|
|
@@ -13212,13 +12226,13 @@ const LinkedInPages = (function() {
|
|
|
13212
12226
|
* @param {Array} options.urns - URNs to process
|
|
13213
12227
|
* @param {Array} options.fields - Fields to fetch
|
|
13214
12228
|
*/
|
|
13215
|
-
processNode({ nodeName, urns, fields }) {
|
|
12229
|
+
async processNode({ nodeName, urns, fields }) {
|
|
13216
12230
|
const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
|
|
13217
12231
|
const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
|
|
13218
12232
|
if (isTimeSeriesNode && !dateInfo) {
|
|
13219
12233
|
return;
|
|
13220
12234
|
}
|
|
13221
|
-
this.fetchAndSaveData({
|
|
12235
|
+
await this.fetchAndSaveData({
|
|
13222
12236
|
nodeName,
|
|
13223
12237
|
urns,
|
|
13224
12238
|
fields,
|
|
@@ -13239,7 +12253,7 @@ const LinkedInPages = (function() {
|
|
|
13239
12253
|
* @param {string} [options.startDate] - Start date for time series data
|
|
13240
12254
|
* @param {string} [options.endDate] - End date for time series data
|
|
13241
12255
|
*/
|
|
13242
|
-
fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
12256
|
+
async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
13243
12257
|
var _a;
|
|
13244
12258
|
for (const urn of urns) {
|
|
13245
12259
|
console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
|
|
@@ -13247,11 +12261,12 @@ const LinkedInPages = (function() {
|
|
|
13247
12261
|
console.log(`End date is +1 day due to LinkedIn Pages API requirements (to include actual end date in results)`);
|
|
13248
12262
|
}
|
|
13249
12263
|
const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
|
|
13250
|
-
const data = this.source.fetchData(nodeName, urn, params);
|
|
12264
|
+
const data = await this.source.fetchData(nodeName, urn, params);
|
|
13251
12265
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
|
|
13252
12266
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
13253
12267
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
13254
|
-
this.getStorageByNode(nodeName)
|
|
12268
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
12269
|
+
await storage.saveData(preparedData);
|
|
13255
12270
|
}
|
|
13256
12271
|
}
|
|
13257
12272
|
}
|
|
@@ -13260,7 +12275,7 @@ const LinkedInPages = (function() {
|
|
|
13260
12275
|
* @param {string} nodeName - Name of the node
|
|
13261
12276
|
* @returns {Object} - Storage instance
|
|
13262
12277
|
*/
|
|
13263
|
-
getStorageByNode(nodeName) {
|
|
12278
|
+
async getStorageByNode(nodeName) {
|
|
13264
12279
|
if (!("storages" in this)) {
|
|
13265
12280
|
this.storages = {};
|
|
13266
12281
|
}
|
|
@@ -13278,6 +12293,7 @@ const LinkedInPages = (function() {
|
|
|
13278
12293
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
13279
12294
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
13280
12295
|
);
|
|
12296
|
+
await this.storages[nodeName].init();
|
|
13281
12297
|
}
|
|
13282
12298
|
return this.storages[nodeName];
|
|
13283
12299
|
}
|
|
@@ -13315,7 +12331,7 @@ const LinkedInPages = (function() {
|
|
|
13315
12331
|
};
|
|
13316
12332
|
})();
|
|
13317
12333
|
const LinkedInAds = (function() {
|
|
13318
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
12334
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
13319
12335
|
var creativesFields = {
|
|
13320
12336
|
"account": {
|
|
13321
12337
|
"description": "URN identifying the advertising account associated with the creative. This field is read-only.",
|
|
@@ -14271,7 +13287,7 @@ const LinkedInAds = (function() {
|
|
|
14271
13287
|
* @param {Object} params - Additional parameters for the request
|
|
14272
13288
|
* @returns {Array} - Array of fetched data objects
|
|
14273
13289
|
*/
|
|
14274
|
-
fetchData(nodeName, urn, params = {}) {
|
|
13290
|
+
async fetchData(nodeName, urn, params = {}) {
|
|
14275
13291
|
var _a;
|
|
14276
13292
|
const fields = params.fields || [];
|
|
14277
13293
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -14281,15 +13297,15 @@ const LinkedInAds = (function() {
|
|
|
14281
13297
|
}
|
|
14282
13298
|
switch (nodeName) {
|
|
14283
13299
|
case "adAccounts":
|
|
14284
|
-
return this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
|
|
13300
|
+
return await this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
|
|
14285
13301
|
case "adCampaignGroups":
|
|
14286
|
-
return this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
|
|
13302
|
+
return await this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
|
|
14287
13303
|
case "adCampaigns":
|
|
14288
|
-
return this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
|
|
13304
|
+
return await this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
|
|
14289
13305
|
case "creatives":
|
|
14290
|
-
return this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
|
|
13306
|
+
return await this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
|
|
14291
13307
|
case "adAnalytics":
|
|
14292
|
-
return this.fetchAdAnalytics(urn, params);
|
|
13308
|
+
return await this.fetchAdAnalytics(urn, params);
|
|
14293
13309
|
default:
|
|
14294
13310
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
14295
13311
|
}
|
|
@@ -14302,10 +13318,10 @@ const LinkedInAds = (function() {
|
|
|
14302
13318
|
* @param {Object} options.params - Additional parameters for the request
|
|
14303
13319
|
* @returns {Array} - Array containing the single resource
|
|
14304
13320
|
*/
|
|
14305
|
-
fetchSingleResource({ urn, resourceType, params }) {
|
|
13321
|
+
async fetchSingleResource({ urn, resourceType, params }) {
|
|
14306
13322
|
let url = `${this.BASE_URL}${resourceType}/${encodeURIComponent(urn)}`;
|
|
14307
13323
|
url += `?fields=${this.formatFields(params.fields)}`;
|
|
14308
|
-
const result = this.makeRequest(url);
|
|
13324
|
+
const result = await this.makeRequest(url);
|
|
14309
13325
|
return [result];
|
|
14310
13326
|
}
|
|
14311
13327
|
/**
|
|
@@ -14317,10 +13333,10 @@ const LinkedInAds = (function() {
|
|
|
14317
13333
|
* @param {string} [options.queryType='search'] - Query type parameter
|
|
14318
13334
|
* @returns {Array} - Array of fetched resources
|
|
14319
13335
|
*/
|
|
14320
|
-
fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
|
|
13336
|
+
async fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
|
|
14321
13337
|
let url = `${this.BASE_URL}adAccounts/${encodeURIComponent(urn)}/${resourceType}?q=${queryType}&pageSize=100`;
|
|
14322
13338
|
url += `&fields=${this.formatFields(params.fields)}`;
|
|
14323
|
-
return this.fetchWithPagination(url);
|
|
13339
|
+
return await this.fetchWithPagination(url);
|
|
14324
13340
|
}
|
|
14325
13341
|
/**
|
|
14326
13342
|
* Fetch analytics data, handling field limits and data merging
|
|
@@ -14331,7 +13347,7 @@ const LinkedInAds = (function() {
|
|
|
14331
13347
|
* @param {Array} params.fields - Fields to fetch
|
|
14332
13348
|
* @returns {Array} - Combined array of analytics data
|
|
14333
13349
|
*/
|
|
14334
|
-
fetchAdAnalytics(urn, params) {
|
|
13350
|
+
async fetchAdAnalytics(urn, params) {
|
|
14335
13351
|
const startDate = new Date(params.startDate);
|
|
14336
13352
|
const endDate = new Date(params.endDate);
|
|
14337
13353
|
const accountUrn = `urn:li:sponsoredAccount:${urn}`;
|
|
@@ -14346,7 +13362,7 @@ const LinkedInAds = (function() {
|
|
|
14346
13362
|
encodedUrn,
|
|
14347
13363
|
fields: fieldChunk
|
|
14348
13364
|
});
|
|
14349
|
-
const res = this.makeRequest(url);
|
|
13365
|
+
const res = await this.makeRequest(url);
|
|
14350
13366
|
const elements = res.elements || [];
|
|
14351
13367
|
allResults = this.mergeAnalyticsResults(allResults, elements);
|
|
14352
13368
|
}
|
|
@@ -14475,9 +13491,9 @@ const LinkedInAds = (function() {
|
|
|
14475
13491
|
* @param {Object} headers - Optional additional headers
|
|
14476
13492
|
* @returns {Object} - API response parsed from JSON
|
|
14477
13493
|
*/
|
|
14478
|
-
makeRequest(url) {
|
|
13494
|
+
async makeRequest(url) {
|
|
14479
13495
|
console.log(`LinkedIn Ads API Request URL:`, url);
|
|
14480
|
-
OAuthUtils.getAccessToken({
|
|
13496
|
+
await OAuthUtils.getAccessToken({
|
|
14481
13497
|
config: this.config,
|
|
14482
13498
|
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
|
|
14483
13499
|
formData: {
|
|
@@ -14492,8 +13508,9 @@ const LinkedInAds = (function() {
|
|
|
14492
13508
|
"X-RestLi-Protocol-Version": "2.0.0"
|
|
14493
13509
|
};
|
|
14494
13510
|
const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
|
|
14495
|
-
const response =
|
|
14496
|
-
const
|
|
13511
|
+
const response = await HttpUtils3.fetch(authUrl, { headers });
|
|
13512
|
+
const text = await response.getContentText();
|
|
13513
|
+
const result = JSON.parse(text);
|
|
14497
13514
|
return result;
|
|
14498
13515
|
}
|
|
14499
13516
|
/**
|
|
@@ -14502,7 +13519,7 @@ const LinkedInAds = (function() {
|
|
|
14502
13519
|
* @param {Object} headers - Optional additional headers
|
|
14503
13520
|
* @returns {Array} - Combined array of results from all pages
|
|
14504
13521
|
*/
|
|
14505
|
-
fetchWithPagination(baseUrl) {
|
|
13522
|
+
async fetchWithPagination(baseUrl) {
|
|
14506
13523
|
let allResults = [];
|
|
14507
13524
|
let pageToken = null;
|
|
14508
13525
|
do {
|
|
@@ -14510,7 +13527,7 @@ const LinkedInAds = (function() {
|
|
|
14510
13527
|
if (pageToken) {
|
|
14511
13528
|
pageUrl += `${pageUrl.includes("?") ? "&" : "?"}pageToken=${encodeURIComponent(pageToken)}`;
|
|
14512
13529
|
}
|
|
14513
|
-
const res = this.makeRequest(pageUrl);
|
|
13530
|
+
const res = await this.makeRequest(pageUrl);
|
|
14514
13531
|
const elements = res.elements || [];
|
|
14515
13532
|
allResults = allResults.concat(elements);
|
|
14516
13533
|
const metadata = res.metadata || {};
|
|
@@ -14520,7 +13537,7 @@ const LinkedInAds = (function() {
|
|
|
14520
13537
|
}
|
|
14521
13538
|
};
|
|
14522
13539
|
var LinkedInAdsConnector = class LinkedInAdsConnector extends AbstractConnector3 {
|
|
14523
|
-
constructor(config, source, storageName = "
|
|
13540
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
14524
13541
|
super(config, source, null, runConfig);
|
|
14525
13542
|
this.storageName = storageName;
|
|
14526
13543
|
}
|
|
@@ -14528,11 +13545,11 @@ const LinkedInAds = (function() {
|
|
|
14528
13545
|
* Main method - entry point for the import process
|
|
14529
13546
|
* Processes all nodes defined in the fields configuration
|
|
14530
13547
|
*/
|
|
14531
|
-
startImportProcess() {
|
|
13548
|
+
async startImportProcess() {
|
|
14532
13549
|
const urns = FormatUtils.parseIds(this.config.AccountURNs.value, { prefix: "urn:li:sponsoredAccount:" });
|
|
14533
13550
|
const dataSources = FormatUtils.parseFields(this.config.Fields.value);
|
|
14534
13551
|
for (const nodeName in dataSources) {
|
|
14535
|
-
this.processNode({
|
|
13552
|
+
await this.processNode({
|
|
14536
13553
|
nodeName,
|
|
14537
13554
|
urns,
|
|
14538
13555
|
fields: dataSources[nodeName] || []
|
|
@@ -14546,13 +13563,13 @@ const LinkedInAds = (function() {
|
|
|
14546
13563
|
* @param {Array} options.urns - URNs to process
|
|
14547
13564
|
* @param {Array} options.fields - Fields to fetch
|
|
14548
13565
|
*/
|
|
14549
|
-
processNode({ nodeName, urns, fields }) {
|
|
13566
|
+
async processNode({ nodeName, urns, fields }) {
|
|
14550
13567
|
const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
|
|
14551
13568
|
const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
|
|
14552
13569
|
if (isTimeSeriesNode && !dateInfo) {
|
|
14553
13570
|
return;
|
|
14554
13571
|
}
|
|
14555
|
-
this.fetchAndSaveData({
|
|
13572
|
+
await this.fetchAndSaveData({
|
|
14556
13573
|
nodeName,
|
|
14557
13574
|
urns,
|
|
14558
13575
|
fields,
|
|
@@ -14573,16 +13590,17 @@ const LinkedInAds = (function() {
|
|
|
14573
13590
|
* @param {string} [options.startDate] - Start date for time series data
|
|
14574
13591
|
* @param {string} [options.endDate] - End date for time series data
|
|
14575
13592
|
*/
|
|
14576
|
-
fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
13593
|
+
async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
14577
13594
|
var _a;
|
|
14578
13595
|
for (const urn of urns) {
|
|
14579
13596
|
console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
|
|
14580
13597
|
const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
|
|
14581
|
-
const data = this.source.fetchData(nodeName, urn, params);
|
|
13598
|
+
const data = await this.source.fetchData(nodeName, urn, params);
|
|
14582
13599
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
|
|
14583
13600
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
14584
13601
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
14585
|
-
this.getStorageByNode(nodeName)
|
|
13602
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
13603
|
+
await storage.saveData(preparedData);
|
|
14586
13604
|
}
|
|
14587
13605
|
}
|
|
14588
13606
|
}
|
|
@@ -14591,7 +13609,7 @@ const LinkedInAds = (function() {
|
|
|
14591
13609
|
* @param {string} nodeName - Name of the node
|
|
14592
13610
|
* @returns {Object} - Storage instance
|
|
14593
13611
|
*/
|
|
14594
|
-
getStorageByNode(nodeName) {
|
|
13612
|
+
async getStorageByNode(nodeName) {
|
|
14595
13613
|
if (!("storages" in this)) {
|
|
14596
13614
|
this.storages = {};
|
|
14597
13615
|
}
|
|
@@ -14609,6 +13627,7 @@ const LinkedInAds = (function() {
|
|
|
14609
13627
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
14610
13628
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
14611
13629
|
);
|
|
13630
|
+
await this.storages[nodeName].init();
|
|
14612
13631
|
}
|
|
14613
13632
|
return this.storages[nodeName];
|
|
14614
13633
|
}
|
|
@@ -14645,7 +13664,7 @@ const LinkedInAds = (function() {
|
|
|
14645
13664
|
};
|
|
14646
13665
|
})();
|
|
14647
13666
|
const GoogleAds = (function() {
|
|
14648
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
13667
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
14649
13668
|
var keywordStatsFields = {
|
|
14650
13669
|
"keyword_id": {
|
|
14651
13670
|
"description": "Keyword Criterion ID",
|
|
@@ -15546,7 +14565,7 @@ const GoogleAds = (function() {
|
|
|
15546
14565
|
* Get access token based on authentication type
|
|
15547
14566
|
* Supports OAuth2 and Service Account authentication
|
|
15548
14567
|
*/
|
|
15549
|
-
getAccessToken() {
|
|
14568
|
+
async getAccessToken() {
|
|
15550
14569
|
var _a;
|
|
15551
14570
|
if (this.accessToken && this.tokenExpiryTime && Date.now() < this.tokenExpiryTime) {
|
|
15552
14571
|
return this.accessToken;
|
|
@@ -15559,7 +14578,7 @@ const GoogleAds = (function() {
|
|
|
15559
14578
|
let accessToken;
|
|
15560
14579
|
try {
|
|
15561
14580
|
if (authType === "oauth2") {
|
|
15562
|
-
accessToken = OAuthUtils.getAccessToken({
|
|
14581
|
+
accessToken = await OAuthUtils.getAccessToken({
|
|
15563
14582
|
config: this.config,
|
|
15564
14583
|
tokenUrl: "https://oauth2.googleapis.com/token",
|
|
15565
14584
|
formData: {
|
|
@@ -15570,7 +14589,7 @@ const GoogleAds = (function() {
|
|
|
15570
14589
|
}
|
|
15571
14590
|
});
|
|
15572
14591
|
} else if (authType === "service_account") {
|
|
15573
|
-
accessToken = OAuthUtils.getServiceAccountToken({
|
|
14592
|
+
accessToken = await OAuthUtils.getServiceAccountToken({
|
|
15574
14593
|
config: this.config,
|
|
15575
14594
|
tokenUrl: "https://oauth2.googleapis.com/token",
|
|
15576
14595
|
serviceAccountKeyJson: authConfig.ServiceAccountKey.value,
|
|
@@ -15597,11 +14616,12 @@ const GoogleAds = (function() {
|
|
|
15597
14616
|
* @param {Date} [options.startDate] - Start date for time series data
|
|
15598
14617
|
* @returns {Array<Object>} - Fetched data
|
|
15599
14618
|
*/
|
|
15600
|
-
fetchData(nodeName, customerId, options) {
|
|
14619
|
+
async fetchData(nodeName, customerId, options) {
|
|
15601
14620
|
console.log("Fetching data from Google Ads API for customer:", customerId);
|
|
15602
14621
|
const { fields, startDate } = options;
|
|
15603
14622
|
const query = this._buildQuery({ nodeName, fields, startDate });
|
|
15604
|
-
|
|
14623
|
+
const response = await this.makeRequest({ customerId, query, nodeName, fields });
|
|
14624
|
+
return await response;
|
|
15605
14625
|
}
|
|
15606
14626
|
/**
|
|
15607
14627
|
* Convert field names to API field names
|
|
@@ -15651,7 +14671,7 @@ const GoogleAds = (function() {
|
|
|
15651
14671
|
const resourceName = this._getResourceName(nodeName);
|
|
15652
14672
|
let query = `SELECT ${apiFields.join(", ")} FROM ${resourceName}`;
|
|
15653
14673
|
if (startDate && this.fieldsSchema[nodeName].isTimeSeries) {
|
|
15654
|
-
const formattedDate =
|
|
14674
|
+
const formattedDate = DateUtils3.formatDate(startDate);
|
|
15655
14675
|
query += ` WHERE segments.date = '${formattedDate}'`;
|
|
15656
14676
|
}
|
|
15657
14677
|
return query;
|
|
@@ -15665,9 +14685,9 @@ const GoogleAds = (function() {
|
|
|
15665
14685
|
* @param {Array<string>} options.fields - Fields that were requested
|
|
15666
14686
|
* @returns {Array<Object>} - API response data
|
|
15667
14687
|
*/
|
|
15668
|
-
makeRequest({ customerId, query, nodeName, fields }) {
|
|
14688
|
+
async makeRequest({ customerId, query, nodeName, fields }) {
|
|
15669
14689
|
var _a, _b;
|
|
15670
|
-
const accessToken = this.getAccessToken();
|
|
14690
|
+
const accessToken = await this.getAccessToken();
|
|
15671
14691
|
const url = `https://googleads.googleapis.com/v21/customers/${customerId}/googleAds:search`;
|
|
15672
14692
|
console.log(`Google Ads API Request URL: ${url}`);
|
|
15673
14693
|
console.log(`GAQL Query: ${query}`);
|
|
@@ -15691,8 +14711,9 @@ const GoogleAds = (function() {
|
|
|
15691
14711
|
body: JSON.stringify(requestBody),
|
|
15692
14712
|
muteHttpExceptions: true
|
|
15693
14713
|
};
|
|
15694
|
-
const response = this.urlFetchWithRetry(url, options);
|
|
15695
|
-
const
|
|
14714
|
+
const response = await this.urlFetchWithRetry(url, options);
|
|
14715
|
+
const text = await response.getContentText();
|
|
14716
|
+
const jsonData = JSON.parse(text);
|
|
15696
14717
|
if (jsonData.error) {
|
|
15697
14718
|
throw new Error(`Google Ads API error: ${jsonData.error.message}`);
|
|
15698
14719
|
}
|
|
@@ -15747,7 +14768,7 @@ const GoogleAds = (function() {
|
|
|
15747
14768
|
}
|
|
15748
14769
|
};
|
|
15749
14770
|
var GoogleAdsConnector = class GoogleAdsConnector extends AbstractConnector3 {
|
|
15750
|
-
constructor(config, source, storageName = "
|
|
14771
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
15751
14772
|
super(config, source, null, runConfig);
|
|
15752
14773
|
this.storageName = storageName;
|
|
15753
14774
|
}
|
|
@@ -15755,11 +14776,11 @@ const GoogleAds = (function() {
|
|
|
15755
14776
|
* Main method - entry point for the import process
|
|
15756
14777
|
* Processes all nodes defined in the fields configuration
|
|
15757
14778
|
*/
|
|
15758
|
-
startImportProcess() {
|
|
14779
|
+
async startImportProcess() {
|
|
15759
14780
|
const customerIds = FormatUtils.parseIds(this.config.CustomerId.value, { stripCharacters: "-" });
|
|
15760
14781
|
const fields = FormatUtils.parseFields(this.config.Fields.value);
|
|
15761
14782
|
for (const nodeName in fields) {
|
|
15762
|
-
this.processNode({
|
|
14783
|
+
await this.processNode({
|
|
15763
14784
|
nodeName,
|
|
15764
14785
|
customerIds,
|
|
15765
14786
|
fields: fields[nodeName] || []
|
|
@@ -15773,16 +14794,16 @@ const GoogleAds = (function() {
|
|
|
15773
14794
|
* @param {Array<string>} options.customerIds - Array of customer IDs to process
|
|
15774
14795
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15775
14796
|
*/
|
|
15776
|
-
processNode({ nodeName, customerIds, fields }) {
|
|
14797
|
+
async processNode({ nodeName, customerIds, fields }) {
|
|
15777
14798
|
for (const customerId of customerIds) {
|
|
15778
14799
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
15779
|
-
this.processTimeSeriesNode({
|
|
14800
|
+
await this.processTimeSeriesNode({
|
|
15780
14801
|
nodeName,
|
|
15781
14802
|
customerId,
|
|
15782
14803
|
fields
|
|
15783
14804
|
});
|
|
15784
14805
|
} else {
|
|
15785
|
-
this.processCatalogNode({
|
|
14806
|
+
await this.processCatalogNode({
|
|
15786
14807
|
nodeName,
|
|
15787
14808
|
customerId,
|
|
15788
14809
|
fields
|
|
@@ -15797,7 +14818,7 @@ const GoogleAds = (function() {
|
|
|
15797
14818
|
* @param {string} options.customerId - Customer ID
|
|
15798
14819
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15799
14820
|
*/
|
|
15800
|
-
processTimeSeriesNode({ nodeName, customerId, fields }) {
|
|
14821
|
+
async processTimeSeriesNode({ nodeName, customerId, fields }) {
|
|
15801
14822
|
var _a;
|
|
15802
14823
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
15803
14824
|
if (daysToFetch <= 0) {
|
|
@@ -15807,12 +14828,13 @@ const GoogleAds = (function() {
|
|
|
15807
14828
|
for (let i = 0; i < daysToFetch; i++) {
|
|
15808
14829
|
const currentDate = new Date(startDate);
|
|
15809
14830
|
currentDate.setDate(currentDate.getDate() + i);
|
|
15810
|
-
const formattedDate =
|
|
15811
|
-
const data = this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
|
|
14831
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
14832
|
+
const data = await this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
|
|
15812
14833
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId} on ${formattedDate}` : `ℹ️ No records have been fetched`);
|
|
15813
14834
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
15814
14835
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
15815
|
-
this.getStorageByNode(nodeName)
|
|
14836
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
14837
|
+
await storage.saveData(preparedData);
|
|
15816
14838
|
}
|
|
15817
14839
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
15818
14840
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -15826,13 +14848,14 @@ const GoogleAds = (function() {
|
|
|
15826
14848
|
* @param {string} options.customerId - Customer ID
|
|
15827
14849
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15828
14850
|
*/
|
|
15829
|
-
processCatalogNode({ nodeName, customerId, fields }) {
|
|
14851
|
+
async processCatalogNode({ nodeName, customerId, fields }) {
|
|
15830
14852
|
var _a;
|
|
15831
|
-
const data = this.source.fetchData(nodeName, customerId, { fields });
|
|
14853
|
+
const data = await this.source.fetchData(nodeName, customerId, { fields });
|
|
15832
14854
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId}` : `ℹ️ No records have been fetched`);
|
|
15833
14855
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
15834
14856
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
15835
|
-
this.getStorageByNode(nodeName)
|
|
14857
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
14858
|
+
await storage.saveData(preparedData);
|
|
15836
14859
|
}
|
|
15837
14860
|
}
|
|
15838
14861
|
/**
|
|
@@ -15840,7 +14863,7 @@ const GoogleAds = (function() {
|
|
|
15840
14863
|
* @param {string} nodeName - Name of the node
|
|
15841
14864
|
* @returns {Object} - Storage instance
|
|
15842
14865
|
*/
|
|
15843
|
-
getStorageByNode(nodeName) {
|
|
14866
|
+
async getStorageByNode(nodeName) {
|
|
15844
14867
|
if (!("storages" in this)) {
|
|
15845
14868
|
this.storages = {};
|
|
15846
14869
|
}
|
|
@@ -15858,6 +14881,7 @@ const GoogleAds = (function() {
|
|
|
15858
14881
|
this.source.fieldsSchema[nodeName].fields,
|
|
15859
14882
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
15860
14883
|
);
|
|
14884
|
+
await this.storages[nodeName].init();
|
|
15861
14885
|
}
|
|
15862
14886
|
return this.storages[nodeName];
|
|
15863
14887
|
}
|
|
@@ -15874,7 +14898,7 @@ const GoogleAds = (function() {
|
|
|
15874
14898
|
};
|
|
15875
14899
|
})();
|
|
15876
14900
|
const GitHub = (function() {
|
|
15877
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
14901
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
15878
14902
|
var repositoryStatsFields = {
|
|
15879
14903
|
date: {
|
|
15880
14904
|
type: "date",
|
|
@@ -16187,16 +15211,16 @@ const GitHub = (function() {
|
|
|
16187
15211
|
* @param {Array<string>} opts.fields
|
|
16188
15212
|
* @returns {Array<Object>}
|
|
16189
15213
|
*/
|
|
16190
|
-
fetchData({ nodeName, fields = [] }) {
|
|
15214
|
+
async fetchData({ nodeName, fields = [] }) {
|
|
16191
15215
|
switch (nodeName) {
|
|
16192
15216
|
case "repository":
|
|
16193
|
-
const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
15217
|
+
const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
16194
15218
|
return this._filterBySchema({ items: [repoData], nodeName, fields });
|
|
16195
15219
|
case "contributors":
|
|
16196
|
-
const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
15220
|
+
const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
16197
15221
|
return this._filterBySchema({ items: contribData, nodeName, fields });
|
|
16198
15222
|
case "repositoryStats":
|
|
16199
|
-
return this._fetchRepositoryStats({ nodeName, fields });
|
|
15223
|
+
return await this._fetchRepositoryStats({ nodeName, fields });
|
|
16200
15224
|
default:
|
|
16201
15225
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
16202
15226
|
}
|
|
@@ -16208,9 +15232,9 @@ const GitHub = (function() {
|
|
|
16208
15232
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16209
15233
|
* @returns {Array} Array of repository statistics data
|
|
16210
15234
|
*/
|
|
16211
|
-
_fetchRepositoryStats({ nodeName, fields }) {
|
|
16212
|
-
const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
16213
|
-
const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
15235
|
+
async _fetchRepositoryStats({ nodeName, fields }) {
|
|
15236
|
+
const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
15237
|
+
const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
16214
15238
|
return this._filterBySchema({
|
|
16215
15239
|
items: [{
|
|
16216
15240
|
"date": new Date((/* @__PURE__ */ new Date()).setHours(0, 0, 0, 0)),
|
|
@@ -16227,30 +15251,32 @@ const GitHub = (function() {
|
|
|
16227
15251
|
* @param {string} options.endpoint - API endpoint path (e.g., "repos/owner/repo")
|
|
16228
15252
|
* @returns {Object} - API response parsed from JSON
|
|
16229
15253
|
*/
|
|
16230
|
-
makeRequest({ endpoint }) {
|
|
16231
|
-
|
|
16232
|
-
|
|
16233
|
-
|
|
16234
|
-
|
|
16235
|
-
|
|
16236
|
-
|
|
16237
|
-
"
|
|
16238
|
-
|
|
16239
|
-
|
|
15254
|
+
async makeRequest({ endpoint }) {
|
|
15255
|
+
try {
|
|
15256
|
+
const baseUrl = "https://api.github.com/";
|
|
15257
|
+
const url = `${baseUrl}${endpoint}`;
|
|
15258
|
+
const response = await HttpUtils3.fetch(url, {
|
|
15259
|
+
"method": "get",
|
|
15260
|
+
"muteHttpExceptions": true,
|
|
15261
|
+
"headers": {
|
|
15262
|
+
"Accept": "application/vnd.github+json",
|
|
15263
|
+
"Authorization": `Bearer ${this.config.AccessToken.value}`,
|
|
15264
|
+
"User-Agent": "owox"
|
|
15265
|
+
}
|
|
15266
|
+
});
|
|
15267
|
+
const text = await response.getContentText();
|
|
15268
|
+
const result = JSON.parse(text);
|
|
15269
|
+
if (result && result.message === "Not Found") {
|
|
15270
|
+
throw new Error(
|
|
15271
|
+
"The repository was not found. The repository name should be in the format: owner/repo"
|
|
15272
|
+
);
|
|
16240
15273
|
}
|
|
16241
|
-
|
|
16242
|
-
|
|
16243
|
-
|
|
16244
|
-
|
|
16245
|
-
|
|
16246
|
-
);
|
|
15274
|
+
return result;
|
|
15275
|
+
} catch (error) {
|
|
15276
|
+
this.config.logMessage(`Error: ${error.message}`);
|
|
15277
|
+
console.error(error.stack);
|
|
15278
|
+
throw error;
|
|
16247
15279
|
}
|
|
16248
|
-
return result;
|
|
16249
|
-
}
|
|
16250
|
-
catch(error) {
|
|
16251
|
-
this.config.logMessage(`Error: ${error.message}`);
|
|
16252
|
-
console.error(error.stack);
|
|
16253
|
-
throw error;
|
|
16254
15280
|
}
|
|
16255
15281
|
/**
|
|
16256
15282
|
* Keep only requestedFields plus any schema-required keys.
|
|
@@ -16276,7 +15302,7 @@ const GitHub = (function() {
|
|
|
16276
15302
|
}
|
|
16277
15303
|
};
|
|
16278
15304
|
var GitHubConnector = class GitHubConnector extends AbstractConnector3 {
|
|
16279
|
-
constructor(config, source, storageName = "
|
|
15305
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
16280
15306
|
super(config, source, null, runConfig);
|
|
16281
15307
|
this.storageName = storageName;
|
|
16282
15308
|
}
|
|
@@ -16284,10 +15310,10 @@ const GitHub = (function() {
|
|
|
16284
15310
|
* Main method - entry point for the import process
|
|
16285
15311
|
* Processes all nodes defined in the fields configuration
|
|
16286
15312
|
*/
|
|
16287
|
-
startImportProcess() {
|
|
15313
|
+
async startImportProcess() {
|
|
16288
15314
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
16289
15315
|
for (const nodeName in fields) {
|
|
16290
|
-
this.processNode({
|
|
15316
|
+
await this.processNode({
|
|
16291
15317
|
nodeName,
|
|
16292
15318
|
fields: fields[nodeName] || []
|
|
16293
15319
|
});
|
|
@@ -16299,11 +15325,11 @@ const GitHub = (function() {
|
|
|
16299
15325
|
* @param {string} options.nodeName - Name of the node to process
|
|
16300
15326
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16301
15327
|
*/
|
|
16302
|
-
processNode({ nodeName, fields }) {
|
|
15328
|
+
async processNode({ nodeName, fields }) {
|
|
16303
15329
|
if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
|
|
16304
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
15330
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
16305
15331
|
} else {
|
|
16306
|
-
this.processCatalogNode({ nodeName, fields });
|
|
15332
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
16307
15333
|
}
|
|
16308
15334
|
}
|
|
16309
15335
|
/**
|
|
@@ -16313,7 +15339,7 @@ const GitHub = (function() {
|
|
|
16313
15339
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16314
15340
|
* @param {Object} options.storage - Storage instance
|
|
16315
15341
|
*/
|
|
16316
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
15342
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
16317
15343
|
console.log(`Time series node processing not implemented for ${nodeName}`);
|
|
16318
15344
|
}
|
|
16319
15345
|
/**
|
|
@@ -16323,13 +15349,14 @@ const GitHub = (function() {
|
|
|
16323
15349
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16324
15350
|
* @param {Object} options.storage - Storage instance
|
|
16325
15351
|
*/
|
|
16326
|
-
processCatalogNode({ nodeName, fields }) {
|
|
15352
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
16327
15353
|
var _a;
|
|
16328
|
-
const data = this.source.fetchData({ nodeName, fields });
|
|
15354
|
+
const data = await this.source.fetchData({ nodeName, fields });
|
|
16329
15355
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched` : `No records have been fetched`);
|
|
16330
15356
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
16331
15357
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
16332
|
-
this.getStorageByNode(nodeName)
|
|
15358
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
15359
|
+
await storage.saveData(preparedData);
|
|
16333
15360
|
}
|
|
16334
15361
|
}
|
|
16335
15362
|
/**
|
|
@@ -16337,7 +15364,7 @@ const GitHub = (function() {
|
|
|
16337
15364
|
* @param {string} nodeName - Name of the node
|
|
16338
15365
|
* @returns {Object} Storage instance
|
|
16339
15366
|
*/
|
|
16340
|
-
getStorageByNode(nodeName) {
|
|
15367
|
+
async getStorageByNode(nodeName) {
|
|
16341
15368
|
if (!("storages" in this)) {
|
|
16342
15369
|
this.storages = {};
|
|
16343
15370
|
}
|
|
@@ -16355,6 +15382,7 @@ const GitHub = (function() {
|
|
|
16355
15382
|
this.source.fieldsSchema[nodeName].fields,
|
|
16356
15383
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
16357
15384
|
);
|
|
15385
|
+
await this.storages[nodeName].init();
|
|
16358
15386
|
}
|
|
16359
15387
|
return this.storages[nodeName];
|
|
16360
15388
|
}
|
|
@@ -16371,7 +15399,7 @@ const GitHub = (function() {
|
|
|
16371
15399
|
};
|
|
16372
15400
|
})();
|
|
16373
15401
|
const FacebookMarketing = (function() {
|
|
16374
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
15402
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
16375
15403
|
var adGroupFields = {
|
|
16376
15404
|
"id": {
|
|
16377
15405
|
"description": "The ID of this ad.",
|
|
@@ -20876,12 +19904,12 @@ const FacebookMarketing = (function() {
|
|
|
20876
19904
|
@return data array
|
|
20877
19905
|
|
|
20878
19906
|
*/
|
|
20879
|
-
fetchData(nodeName, accountId, fields, startDate = null) {
|
|
19907
|
+
async fetchData(nodeName, accountId, fields, startDate = null) {
|
|
20880
19908
|
let url = "https://graph.facebook.com/v23.0/";
|
|
20881
19909
|
let formattedDate = null;
|
|
20882
19910
|
let timeRange = null;
|
|
20883
19911
|
if (startDate) {
|
|
20884
|
-
formattedDate =
|
|
19912
|
+
formattedDate = DateUtils3.formatDate(startDate);
|
|
20885
19913
|
timeRange = encodeURIComponent(JSON.stringify({ since: formattedDate, until: formattedDate }));
|
|
20886
19914
|
}
|
|
20887
19915
|
switch (nodeName) {
|
|
@@ -20905,7 +19933,7 @@ const FacebookMarketing = (function() {
|
|
|
20905
19933
|
case "ad-account/insights-by-region":
|
|
20906
19934
|
case "ad-account/insights-by-product-id":
|
|
20907
19935
|
case "ad-account/insights-by-age-and-gender":
|
|
20908
|
-
return this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
|
|
19936
|
+
return await this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
|
|
20909
19937
|
case "ad-group":
|
|
20910
19938
|
url += `act_${accountId}/ads?fields=${this._buildFieldsString({ nodeName, fields })}&limit=${this.fieldsSchema[nodeName].limit}`;
|
|
20911
19939
|
break;
|
|
@@ -20914,7 +19942,7 @@ const FacebookMarketing = (function() {
|
|
|
20914
19942
|
}
|
|
20915
19943
|
console.log(`Facebook API URL:`, url);
|
|
20916
19944
|
url += `&access_token=${this.config.AccessToken.value}`;
|
|
20917
|
-
return this._fetchPaginatedData(url, nodeName, fields);
|
|
19945
|
+
return await this._fetchPaginatedData(url, nodeName, fields);
|
|
20918
19946
|
}
|
|
20919
19947
|
//---- castRecordFields -------------------------------------------------
|
|
20920
19948
|
/**
|
|
@@ -20976,7 +20004,7 @@ const FacebookMarketing = (function() {
|
|
|
20976
20004
|
* @return {Array} Processed insights data
|
|
20977
20005
|
* @private
|
|
20978
20006
|
*/
|
|
20979
|
-
_fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
|
|
20007
|
+
async _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
|
|
20980
20008
|
const breakdowns = this.fieldsSchema[nodeName].breakdowns || [];
|
|
20981
20009
|
const regularFields = this._prepareFields({ nodeName, fields, breakdowns });
|
|
20982
20010
|
const requestUrl = this._buildInsightsUrl({
|
|
@@ -20987,7 +20015,7 @@ const FacebookMarketing = (function() {
|
|
|
20987
20015
|
nodeName,
|
|
20988
20016
|
url
|
|
20989
20017
|
});
|
|
20990
|
-
const allData = this._fetchPaginatedData(requestUrl, nodeName, fields);
|
|
20018
|
+
const allData = await this._fetchPaginatedData(requestUrl, nodeName, fields);
|
|
20991
20019
|
if (this.config.ProcessShortLinks.value && allData.length > 0 && allData.some((record) => record.link_url_asset)) {
|
|
20992
20020
|
return processShortLinks(allData, {
|
|
20993
20021
|
shortLinkField: "link_url_asset",
|
|
@@ -21128,12 +20156,13 @@ const FacebookMarketing = (function() {
|
|
|
21128
20156
|
* @return {Array} All fetched data
|
|
21129
20157
|
* @private
|
|
21130
20158
|
*/
|
|
21131
|
-
_fetchPaginatedData(initialUrl, nodeName, fields) {
|
|
20159
|
+
async _fetchPaginatedData(initialUrl, nodeName, fields) {
|
|
21132
20160
|
var allData = [];
|
|
21133
20161
|
var nextPageURL = initialUrl;
|
|
21134
20162
|
while (nextPageURL) {
|
|
21135
|
-
var response = this.urlFetchWithRetry(nextPageURL);
|
|
21136
|
-
var
|
|
20163
|
+
var response = await this.urlFetchWithRetry(nextPageURL);
|
|
20164
|
+
var text = await response.getContentText();
|
|
20165
|
+
var jsonData = JSON.parse(text);
|
|
21137
20166
|
if ("data" in jsonData) {
|
|
21138
20167
|
nextPageURL = jsonData.paging ? jsonData.paging.next : null;
|
|
21139
20168
|
jsonData.data.forEach((record, index) => {
|
|
@@ -21156,12 +20185,12 @@ const FacebookMarketing = (function() {
|
|
|
21156
20185
|
};
|
|
21157
20186
|
var FacebookMarketingConnector = class FacebookMarketingConnector extends AbstractConnector3 {
|
|
21158
20187
|
// ---- constructor ------------------------------------
|
|
21159
|
-
constructor(config, source, storageName = "
|
|
20188
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
21160
20189
|
super(config, source, null, runConfig);
|
|
21161
20190
|
this.storageName = storageName;
|
|
21162
20191
|
}
|
|
21163
20192
|
//---- startImportProcess -------------------------------------------------
|
|
21164
|
-
startImportProcess() {
|
|
20193
|
+
async startImportProcess() {
|
|
21165
20194
|
let accountsIds = String(this.config.AccoundIDs.value).split(/[,;]\s*/);
|
|
21166
20195
|
let fields = this.config.Fields.value.split(", ").reduce((acc, pair) => {
|
|
21167
20196
|
let [key, value] = pair.split(" ");
|
|
@@ -21173,7 +20202,7 @@ const FacebookMarketing = (function() {
|
|
|
21173
20202
|
if (nodeName in this.source.fieldsSchema && this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
21174
20203
|
timeSeriesNodes[nodeName] = fields[nodeName];
|
|
21175
20204
|
} else {
|
|
21176
|
-
this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
|
|
20205
|
+
await this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
|
|
21177
20206
|
}
|
|
21178
20207
|
}
|
|
21179
20208
|
if (Object.keys(timeSeriesNodes).length > 0) {
|
|
@@ -21181,27 +20210,28 @@ const FacebookMarketing = (function() {
|
|
|
21181
20210
|
let daysToFetch = null;
|
|
21182
20211
|
[startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
21183
20212
|
if (daysToFetch > 0) {
|
|
21184
|
-
this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
|
|
20213
|
+
await this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
|
|
21185
20214
|
}
|
|
21186
20215
|
}
|
|
21187
20216
|
}
|
|
21188
20217
|
//---- startImportProcessOfCatalogData -------------------------------------------------
|
|
21189
20218
|
/*
|
|
21190
20219
|
|
|
21191
|
-
Imports catalog (not time seriesed) data
|
|
20220
|
+
Imports catalog (not time seriesed) data
|
|
21192
20221
|
|
|
21193
20222
|
@param nodeName string Node name
|
|
21194
20223
|
@param accountsIds array list of account ids
|
|
21195
|
-
@param fields array list of fields
|
|
20224
|
+
@param fields array list of fields
|
|
21196
20225
|
|
|
21197
20226
|
*/
|
|
21198
|
-
startImportProcessOfCatalogData(nodeName, accountIds, fields) {
|
|
20227
|
+
async startImportProcessOfCatalogData(nodeName, accountIds, fields) {
|
|
21199
20228
|
var _a;
|
|
21200
20229
|
for (var i in accountIds) {
|
|
21201
20230
|
let accountId = accountIds[i];
|
|
21202
|
-
let data = this.source.fetchData(nodeName, accountId, fields);
|
|
20231
|
+
let data = await this.source.fetchData(nodeName, accountId, fields);
|
|
21203
20232
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
21204
|
-
this.getStorageByNode(nodeName)
|
|
20233
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
20234
|
+
await storage.saveData(data);
|
|
21205
20235
|
}
|
|
21206
20236
|
data.length && this.config.logMessage(`${data.length} rows of ${nodeName} were fetched for account ${accountId}`);
|
|
21207
20237
|
}
|
|
@@ -21209,23 +20239,24 @@ const FacebookMarketing = (function() {
|
|
|
21209
20239
|
//---- startImportProcessOfTimeSeriesData -------------------------------------------------
|
|
21210
20240
|
/*
|
|
21211
20241
|
|
|
21212
|
-
Imports time series (not catalog) data
|
|
20242
|
+
Imports time series (not catalog) data
|
|
21213
20243
|
|
|
21214
20244
|
@param accountsIds (array) list of account ids
|
|
21215
20245
|
@param timeSeriesNodes (object) of properties, each is array of fields
|
|
21216
|
-
@param startDate (Data) start date
|
|
20246
|
+
@param startDate (Data) start date
|
|
21217
20247
|
@param daysToFetch (integer) days to import
|
|
21218
20248
|
|
|
21219
20249
|
*/
|
|
21220
|
-
startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
|
|
20250
|
+
async startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
|
|
21221
20251
|
var _a;
|
|
21222
20252
|
for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
21223
20253
|
for (let accountId of accountsIds) {
|
|
21224
20254
|
for (var nodeName in timeSeriesNodes) {
|
|
21225
|
-
this.config.logMessage(`Start importing data for ${
|
|
21226
|
-
let data = this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
|
|
20255
|
+
this.config.logMessage(`Start importing data for ${DateUtils3.formatDate(startDate)}: ${accountId}/${nodeName}`);
|
|
20256
|
+
let data = await this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
|
|
21227
20257
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
21228
|
-
this.getStorageByNode(nodeName)
|
|
20258
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
20259
|
+
await storage.saveData(data);
|
|
21229
20260
|
}
|
|
21230
20261
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
21231
20262
|
}
|
|
@@ -21245,7 +20276,7 @@ const FacebookMarketing = (function() {
|
|
|
21245
20276
|
* @return AbstractStorage
|
|
21246
20277
|
*
|
|
21247
20278
|
*/
|
|
21248
|
-
getStorageByNode(nodeName) {
|
|
20279
|
+
async getStorageByNode(nodeName) {
|
|
21249
20280
|
if (!("storages" in this)) {
|
|
21250
20281
|
this.storages = {};
|
|
21251
20282
|
}
|
|
@@ -21263,6 +20294,7 @@ const FacebookMarketing = (function() {
|
|
|
21263
20294
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
21264
20295
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
21265
20296
|
);
|
|
20297
|
+
await this.storages[nodeName].init();
|
|
21266
20298
|
}
|
|
21267
20299
|
return this.storages[nodeName];
|
|
21268
20300
|
}
|
|
@@ -21279,7 +20311,7 @@ const FacebookMarketing = (function() {
|
|
|
21279
20311
|
};
|
|
21280
20312
|
})();
|
|
21281
20313
|
const CriteoAds = (function() {
|
|
21282
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
20314
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
21283
20315
|
var CriteoAdsHelper = {
|
|
21284
20316
|
/**
|
|
21285
20317
|
* Parse fields string into a structured object
|
|
@@ -22126,10 +21158,10 @@ const CriteoAds = (function() {
|
|
|
22126
21158
|
* @param {Date} opts.date
|
|
22127
21159
|
* @returns {Array<Object>}
|
|
22128
21160
|
*/
|
|
22129
|
-
fetchData({ nodeName, accountId, fields = [], date }) {
|
|
21161
|
+
async fetchData({ nodeName, accountId, fields = [], date }) {
|
|
22130
21162
|
switch (nodeName) {
|
|
22131
21163
|
case "statistics":
|
|
22132
|
-
return this._fetchStatistics({ accountId, fields, date });
|
|
21164
|
+
return await this._fetchStatistics({ accountId, fields, date });
|
|
22133
21165
|
default:
|
|
22134
21166
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
22135
21167
|
}
|
|
@@ -22143,16 +21175,17 @@ const CriteoAds = (function() {
|
|
|
22143
21175
|
* @returns {Array<Object>} - Parsed and enriched data
|
|
22144
21176
|
* @private
|
|
22145
21177
|
*/
|
|
22146
|
-
_fetchStatistics({ accountId, fields, date }) {
|
|
21178
|
+
async _fetchStatistics({ accountId, fields, date }) {
|
|
22147
21179
|
const uniqueKeys = this.fieldsSchema.statistics.uniqueKeys || [];
|
|
22148
21180
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
22149
21181
|
if (missingKeys.length > 0) {
|
|
22150
21182
|
throw new Error(`Missing required unique fields for endpoint 'statistics'. Missing fields: ${missingKeys.join(", ")}`);
|
|
22151
21183
|
}
|
|
22152
|
-
this.getAccessToken();
|
|
21184
|
+
await this.getAccessToken();
|
|
22153
21185
|
const requestBody = this._buildStatisticsRequestBody({ accountId, fields, date });
|
|
22154
|
-
const response = this._makeApiRequest(requestBody);
|
|
22155
|
-
const
|
|
21186
|
+
const response = await this._makeApiRequest(requestBody);
|
|
21187
|
+
const text = await response.getContentText();
|
|
21188
|
+
const jsonObject = JSON.parse(text);
|
|
22156
21189
|
return this.parseApiResponse({ apiResponse: jsonObject, date, accountId, fields });
|
|
22157
21190
|
}
|
|
22158
21191
|
/**
|
|
@@ -22190,7 +21223,7 @@ const CriteoAds = (function() {
|
|
|
22190
21223
|
* @returns {Object} - HTTP response
|
|
22191
21224
|
* @private
|
|
22192
21225
|
*/
|
|
22193
|
-
_makeApiRequest(requestBody) {
|
|
21226
|
+
async _makeApiRequest(requestBody) {
|
|
22194
21227
|
const apiVersion = "2025-07";
|
|
22195
21228
|
const apiUrl = `https://api.criteo.com/${apiVersion}/statistics/report`;
|
|
22196
21229
|
const options = {
|
|
@@ -22204,19 +21237,20 @@ const CriteoAds = (function() {
|
|
|
22204
21237
|
body: JSON.stringify(requestBody)
|
|
22205
21238
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
22206
21239
|
};
|
|
22207
|
-
const response = this.urlFetchWithRetry(apiUrl, options);
|
|
21240
|
+
const response = await this.urlFetchWithRetry(apiUrl, options);
|
|
22208
21241
|
const responseCode = response.getResponseCode();
|
|
22209
21242
|
if (responseCode === HTTP_STATUS2.OK) {
|
|
22210
21243
|
return response;
|
|
22211
21244
|
} else {
|
|
22212
|
-
|
|
21245
|
+
const text = await response.getContentText();
|
|
21246
|
+
throw new Error(`API Error (${responseCode}): ${text}`);
|
|
22213
21247
|
}
|
|
22214
21248
|
}
|
|
22215
21249
|
/**
|
|
22216
21250
|
* Get access token from API
|
|
22217
21251
|
* Docs: https://developers.criteo.com/marketing-solutions/docs/authorization-code-setup
|
|
22218
21252
|
*/
|
|
22219
|
-
getAccessToken() {
|
|
21253
|
+
async getAccessToken() {
|
|
22220
21254
|
var _a;
|
|
22221
21255
|
if ((_a = this.config.AccessToken) == null ? void 0 : _a.value) {
|
|
22222
21256
|
return;
|
|
@@ -22239,8 +21273,9 @@ const CriteoAds = (function() {
|
|
|
22239
21273
|
muteHttpExceptions: true
|
|
22240
21274
|
};
|
|
22241
21275
|
try {
|
|
22242
|
-
const response = this.urlFetchWithRetry(tokenUrl, options);
|
|
22243
|
-
const
|
|
21276
|
+
const response = await this.urlFetchWithRetry(tokenUrl, options);
|
|
21277
|
+
const text = await response.getContentText();
|
|
21278
|
+
const responseData = JSON.parse(text);
|
|
22244
21279
|
const accessToken = responseData["access_token"];
|
|
22245
21280
|
this.config.AccessToken = {
|
|
22246
21281
|
value: accessToken
|
|
@@ -22288,20 +21323,20 @@ const CriteoAds = (function() {
|
|
|
22288
21323
|
}
|
|
22289
21324
|
};
|
|
22290
21325
|
var CriteoAdsConnector = class CriteoAdsConnector extends AbstractConnector3 {
|
|
22291
|
-
constructor(config, source, storageName = "
|
|
21326
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
22292
21327
|
super(config, source, null, runConfig);
|
|
22293
21328
|
this.storageName = storageName;
|
|
22294
21329
|
}
|
|
22295
21330
|
/**
|
|
22296
21331
|
* Main method - entry point for the import process
|
|
22297
21332
|
*/
|
|
22298
|
-
startImportProcess() {
|
|
21333
|
+
async startImportProcess() {
|
|
22299
21334
|
var _a, _b;
|
|
22300
21335
|
const fields = CriteoAdsHelper.parseFields(((_a = this.config.Fields) == null ? void 0 : _a.value) || "");
|
|
22301
21336
|
const advertiserIds = CriteoAdsHelper.parseAdvertiserIds(((_b = this.config.AdvertiserIDs) == null ? void 0 : _b.value) || "");
|
|
22302
21337
|
for (const advertiserId of advertiserIds) {
|
|
22303
21338
|
for (const nodeName in fields) {
|
|
22304
|
-
this.processNode({
|
|
21339
|
+
await this.processNode({
|
|
22305
21340
|
nodeName,
|
|
22306
21341
|
advertiserId,
|
|
22307
21342
|
fields: fields[nodeName] || []
|
|
@@ -22316,8 +21351,8 @@ const CriteoAds = (function() {
|
|
|
22316
21351
|
* @param {string} options.advertiserId - Advertiser ID
|
|
22317
21352
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22318
21353
|
*/
|
|
22319
|
-
processNode({ nodeName, advertiserId, fields }) {
|
|
22320
|
-
this.processTimeSeriesNode({
|
|
21354
|
+
async processNode({ nodeName, advertiserId, fields }) {
|
|
21355
|
+
await this.processTimeSeriesNode({
|
|
22321
21356
|
nodeName,
|
|
22322
21357
|
advertiserId,
|
|
22323
21358
|
fields
|
|
@@ -22331,7 +21366,7 @@ const CriteoAds = (function() {
|
|
|
22331
21366
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22332
21367
|
* @param {Object} options.storage - Storage instance
|
|
22333
21368
|
*/
|
|
22334
|
-
processTimeSeriesNode({ nodeName, advertiserId, fields }) {
|
|
21369
|
+
async processTimeSeriesNode({ nodeName, advertiserId, fields }) {
|
|
22335
21370
|
var _a;
|
|
22336
21371
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
22337
21372
|
if (daysToFetch <= 0) {
|
|
@@ -22341,8 +21376,8 @@ const CriteoAds = (function() {
|
|
|
22341
21376
|
for (let i = 0; i < daysToFetch; i++) {
|
|
22342
21377
|
const currentDate = new Date(startDate);
|
|
22343
21378
|
currentDate.setDate(currentDate.getDate() + i);
|
|
22344
|
-
const formattedDate =
|
|
22345
|
-
const data = this.source.fetchData({
|
|
21379
|
+
const formattedDate = DateUtils3.formatDate(currentDate);
|
|
21380
|
+
const data = await this.source.fetchData({
|
|
22346
21381
|
nodeName,
|
|
22347
21382
|
accountId: advertiserId,
|
|
22348
21383
|
date: currentDate,
|
|
@@ -22351,7 +21386,8 @@ const CriteoAds = (function() {
|
|
|
22351
21386
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${advertiserId} on ${formattedDate}` : `No records have been fetched`);
|
|
22352
21387
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
22353
21388
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
22354
|
-
this.getStorageByNode(nodeName)
|
|
21389
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
21390
|
+
await storage.saveData(preparedData);
|
|
22355
21391
|
}
|
|
22356
21392
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
22357
21393
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -22363,7 +21399,7 @@ const CriteoAds = (function() {
|
|
|
22363
21399
|
* @param {string} nodeName - Name of the node
|
|
22364
21400
|
* @returns {Object} Storage instance
|
|
22365
21401
|
*/
|
|
22366
|
-
getStorageByNode(nodeName) {
|
|
21402
|
+
async getStorageByNode(nodeName) {
|
|
22367
21403
|
if (!("storages" in this)) {
|
|
22368
21404
|
this.storages = {};
|
|
22369
21405
|
}
|
|
@@ -22381,6 +21417,7 @@ const CriteoAds = (function() {
|
|
|
22381
21417
|
this.source.fieldsSchema[nodeName].fields,
|
|
22382
21418
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
22383
21419
|
);
|
|
21420
|
+
await this.storages[nodeName].init();
|
|
22384
21421
|
}
|
|
22385
21422
|
return this.storages[nodeName];
|
|
22386
21423
|
}
|
|
@@ -22397,7 +21434,7 @@ const CriteoAds = (function() {
|
|
|
22397
21434
|
};
|
|
22398
21435
|
})();
|
|
22399
21436
|
const BankOfCanada = (function() {
|
|
22400
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
21437
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
22401
21438
|
var observationsFields = {
|
|
22402
21439
|
"date": {
|
|
22403
21440
|
"description": "The date for which the exchange rate was recorded.",
|
|
@@ -22475,10 +21512,10 @@ const BankOfCanada = (function() {
|
|
|
22475
21512
|
* @param {string} [opts.end_time]
|
|
22476
21513
|
* @returns {Array<Object>}
|
|
22477
21514
|
*/
|
|
22478
|
-
fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
21515
|
+
async fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
22479
21516
|
switch (nodeName) {
|
|
22480
21517
|
case "observations/group":
|
|
22481
|
-
return this._fetchObservations({ fields, start_time, end_time });
|
|
21518
|
+
return await this._fetchObservations({ fields, start_time, end_time });
|
|
22482
21519
|
default:
|
|
22483
21520
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
22484
21521
|
}
|
|
@@ -22491,8 +21528,8 @@ const BankOfCanada = (function() {
|
|
|
22491
21528
|
* @param {string} opts.end_time
|
|
22492
21529
|
* @returns {Array<Object>}
|
|
22493
21530
|
*/
|
|
22494
|
-
_fetchObservations({ fields, start_time, end_time }) {
|
|
22495
|
-
const rates = this.makeRequest({
|
|
21531
|
+
async _fetchObservations({ fields, start_time, end_time }) {
|
|
21532
|
+
const rates = await this.makeRequest({
|
|
22496
21533
|
endpoint: `observations/group/FX_RATES_DAILY/json?start_date=${start_time}&end_date=${end_time}`
|
|
22497
21534
|
});
|
|
22498
21535
|
let data = [];
|
|
@@ -22514,13 +21551,13 @@ const BankOfCanada = (function() {
|
|
|
22514
21551
|
* @param {string} options.endpoint - API endpoint path (e.g., "observations/group/FX_RATES_DAILY/json")
|
|
22515
21552
|
* @returns {Object} - API response parsed from JSON
|
|
22516
21553
|
*/
|
|
22517
|
-
makeRequest({ endpoint }) {
|
|
21554
|
+
async makeRequest({ endpoint }) {
|
|
22518
21555
|
const baseUrl = "https://www.bankofcanada.ca/valet/";
|
|
22519
21556
|
const url = `${baseUrl}${endpoint}`;
|
|
22520
21557
|
console.log(`Bank of Canada API Request URL:`, url);
|
|
22521
|
-
const response =
|
|
22522
|
-
const result =
|
|
22523
|
-
return result;
|
|
21558
|
+
const response = await HttpUtils3.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
21559
|
+
const result = await response.getContentText();
|
|
21560
|
+
return JSON.parse(result);
|
|
22524
21561
|
}
|
|
22525
21562
|
/**
|
|
22526
21563
|
* Keep only requestedFields plus any schema-required keys.
|
|
@@ -22545,7 +21582,7 @@ const BankOfCanada = (function() {
|
|
|
22545
21582
|
}
|
|
22546
21583
|
};
|
|
22547
21584
|
var BankOfCanadaConnector = class BankOfCanadaConnector extends AbstractConnector3 {
|
|
22548
|
-
constructor(config, source, storageName = "
|
|
21585
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
22549
21586
|
super(config, source, null, runConfig);
|
|
22550
21587
|
this.storageName = storageName;
|
|
22551
21588
|
}
|
|
@@ -22553,10 +21590,10 @@ const BankOfCanada = (function() {
|
|
|
22553
21590
|
* Main method - entry point for the import process
|
|
22554
21591
|
* Processes all nodes defined in the fields configuration
|
|
22555
21592
|
*/
|
|
22556
|
-
startImportProcess() {
|
|
21593
|
+
async startImportProcess() {
|
|
22557
21594
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
22558
21595
|
for (const nodeName in fields) {
|
|
22559
|
-
this.processNode({
|
|
21596
|
+
await this.processNode({
|
|
22560
21597
|
nodeName,
|
|
22561
21598
|
fields: fields[nodeName] || []
|
|
22562
21599
|
});
|
|
@@ -22568,11 +21605,11 @@ const BankOfCanada = (function() {
|
|
|
22568
21605
|
* @param {string} options.nodeName - Name of the node to process
|
|
22569
21606
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22570
21607
|
*/
|
|
22571
|
-
processNode({ nodeName, fields }) {
|
|
21608
|
+
async processNode({ nodeName, fields }) {
|
|
22572
21609
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
22573
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
21610
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
22574
21611
|
} else {
|
|
22575
|
-
this.processCatalogNode({ nodeName, fields });
|
|
21612
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
22576
21613
|
}
|
|
22577
21614
|
}
|
|
22578
21615
|
/**
|
|
@@ -22582,14 +21619,14 @@ const BankOfCanada = (function() {
|
|
|
22582
21619
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22583
21620
|
* @param {Object} options.storage - Storage instance
|
|
22584
21621
|
*/
|
|
22585
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
21622
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
22586
21623
|
var _a;
|
|
22587
21624
|
const dateRange = this.prepareDateRange();
|
|
22588
21625
|
if (!dateRange) {
|
|
22589
21626
|
console.log("No days to fetch for time series data");
|
|
22590
21627
|
return;
|
|
22591
21628
|
}
|
|
22592
|
-
const data = this.source.fetchData({
|
|
21629
|
+
const data = await this.source.fetchData({
|
|
22593
21630
|
nodeName,
|
|
22594
21631
|
start_time: dateRange.startDate,
|
|
22595
21632
|
end_time: dateRange.endDate,
|
|
@@ -22598,7 +21635,8 @@ const BankOfCanada = (function() {
|
|
|
22598
21635
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
|
|
22599
21636
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
22600
21637
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
22601
|
-
this.getStorageByNode(nodeName)
|
|
21638
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
21639
|
+
await storage.saveData(preparedData);
|
|
22602
21640
|
}
|
|
22603
21641
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
22604
21642
|
this.config.updateLastRequstedDate(new Date(dateRange.endDate));
|
|
@@ -22611,7 +21649,7 @@ const BankOfCanada = (function() {
|
|
|
22611
21649
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22612
21650
|
* @param {Object} options.storage - Storage instance
|
|
22613
21651
|
*/
|
|
22614
|
-
processCatalogNode({ nodeName, fields }) {
|
|
21652
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
22615
21653
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
22616
21654
|
}
|
|
22617
21655
|
/**
|
|
@@ -22619,7 +21657,7 @@ const BankOfCanada = (function() {
|
|
|
22619
21657
|
* @param {string} nodeName - Name of the node
|
|
22620
21658
|
* @returns {Object} Storage instance
|
|
22621
21659
|
*/
|
|
22622
|
-
getStorageByNode(nodeName) {
|
|
21660
|
+
async getStorageByNode(nodeName) {
|
|
22623
21661
|
if (!("storages" in this)) {
|
|
22624
21662
|
this.storages = {};
|
|
22625
21663
|
}
|
|
@@ -22637,6 +21675,7 @@ const BankOfCanada = (function() {
|
|
|
22637
21675
|
this.source.fieldsSchema[nodeName].fields,
|
|
22638
21676
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
22639
21677
|
);
|
|
21678
|
+
await this.storages[nodeName].init();
|
|
22640
21679
|
}
|
|
22641
21680
|
return this.storages[nodeName];
|
|
22642
21681
|
}
|
|
@@ -22652,8 +21691,8 @@ const BankOfCanada = (function() {
|
|
|
22652
21691
|
const endDate = new Date(startDate);
|
|
22653
21692
|
endDate.setDate(endDate.getDate() + daysToFetch - 1);
|
|
22654
21693
|
return {
|
|
22655
|
-
startDate:
|
|
22656
|
-
endDate:
|
|
21694
|
+
startDate: DateUtils3.formatDate(startDate),
|
|
21695
|
+
endDate: DateUtils3.formatDate(endDate)
|
|
22657
21696
|
};
|
|
22658
21697
|
}
|
|
22659
21698
|
};
|
|
@@ -22699,7 +21738,6 @@ const AvailableConnectors = [
|
|
|
22699
21738
|
"BankOfCanada"
|
|
22700
21739
|
];
|
|
22701
21740
|
const AvailableStorages = [
|
|
22702
|
-
"GoogleSheets",
|
|
22703
21741
|
"GoogleBigQuery",
|
|
22704
21742
|
"AwsAthena"
|
|
22705
21743
|
];
|
|
@@ -22724,7 +21762,6 @@ const OWOX = {
|
|
|
22724
21762
|
CriteoAds,
|
|
22725
21763
|
BankOfCanada,
|
|
22726
21764
|
// Individual storages
|
|
22727
|
-
GoogleSheets,
|
|
22728
21765
|
GoogleBigQuery,
|
|
22729
21766
|
AwsAthena
|
|
22730
21767
|
};
|